{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST example for learning rate finder with gradient accumulation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/davidtvs-nb/lr-finder-rev/py3/pytorch-lr-finder/env/lib/python3.6/site-packages/torch_lr_finder/lr_finder.py:5: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
      "  from tqdm.autonotebook import tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import Subset, DataLoader\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "try:\n",
    "    from torch_lr_finder import LRFinder\n",
    "except ImportError:\n",
    "    # Run from source\n",
    "    import sys\n",
    "    sys.path.insert(0, '..')\n",
    "    from torch_lr_finder import LRFinder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import os\n",
    "import numpy as np\n",
    "\n",
    "SEED = 0\n",
    "\n",
    "def reset_seed(seed):\n",
    "    \"\"\"\n",
    "    ref: https://forums.fast.ai/t/accumulating-gradients/33219/28\n",
    "    \"\"\"\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(ConvNet, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)\n",
    "        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)\n",
    "        self.conv2_drop = nn.Dropout2d()\n",
    "        self.fc1 = nn.Linear(320, 50)\n",
    "        self.fc2 = nn.Linear(50, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(F.max_pool2d(self.conv1(x), 2))\n",
    "        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))\n",
    "        x = x.view(-1, 320)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.dropout(x, training=self.training)\n",
    "        x = self.fc2(x)\n",
    "        return F.log_softmax(x, dim=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_folder = '../data'\n",
    "\n",
    "transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.1307,), (0.3081,))\n",
    "])\n",
    "\n",
    "trainset = datasets.MNIST(data_folder, train=True, download=True, transform=transform)\n",
    "testset = datasets.MNIST(data_folder, train=False, download=True, transform=transform)\n",
    "\n",
    "DESIRED_BATCH_SIZE = 256"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4574020fab9a41d48f0dd2b80cb4ec8a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Learning rate search finished. See the graph with {finder_name}.plot()\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEKCAYAAAAMzhLIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deXxU9b3/8ddnMpM9RAhhMeygCMhqVBQrdBO1rRS1/Fzq3lottnptrdfe9rbXXtvb2tLWalXc0Bbbe1Wq1q5qsRRRZBUURFlcWDRhT1gy2/f3xwwpjcmQQM6cnMn7+XjM4zFz5nvOfDjfYT75fr/nfL/mnENERKQlIb8DEBGRjk2JQkREMlKiEBGRjJQoREQkIyUKERHJSIlCREQyCvsdQFt1797dDRgwwO8wREQCZcmSJVudc5WHs2/gEsWAAQNYvHix32GIiASKmb1zuPuq60lERDJSohARkYyUKEREJCMlChERyUiJQkREMlKiEBGRjJQoRMQXqzbvpiGe8DsMaQUlChHJuh17onzmzvnc9NgKv0ORVlCiEJGse/ODOhJJx9Ovbuap5Zv8DkcOQYlCRLJubW09AEN6lPKt373Gxh17fY5IMlGiEJGsW1tTT1Ekjwcuq8YBN/7fqySSWpa5o1KiEJGsW1tTz+AeJfSvKOG754zglQ3bmTlvvd9hSQuUKEQk69bV1DOkshSA88ZVcfbIXvzkr2tY9u4OnyOT5ihRiEhW7WmIs3nXfob0SCUKM+MHU0fRs0sh1z26jJ17oz5HKE0pUYhIVq07aCD7gPLiCHddPI6auv18/bEVOKfxio5EiUJEsmptzYcTBcCYvkdxy1nDeG71Bzwwf4MfoUkLlChEJKvW1tQTDhn9K0o+9N4VEwYweURP/udPb7BU4xUdhhKFiGTV2pp6+lcUE8n78M+PmfGj80fT+6hCps9eyrb6Bh8ilKaUKEQkq9bV1n+o2+lg5UUR7r74BLbtiXL9b5fr/ooOwLNEYWZ9zWyuma0ys9fN7PpmypSb2e/N7NV0mSu8ikdE/BdLJHln214GV7acKACOryrnv6ccz/y1W5nx7JosRSct8bJFEQe+5pwbDowHppvZ8CZlpgOrnHOjgUnAT8ws38OYRMRH72zbQzzpMrYoDph2Yl8uOLEvd81dx7OrPshCdNISzxKFc26Lc25p+nkdsBqoaloMKDMzA0qB7aQSjIjkoJaueGrJd88Zwciqcm78v+Vs2LrHy9Akg6yMUZjZAGAssLDJW3cCw4DNwErgeudcspn9rzazxWa2uLa21uNoRcQrBxLFobqeDiiM5PHLi8cRDhlXPbyIXXtjXoYnLfA8UZhZKfAEcINzbneTtycDy4GjgTHAnWbWpekxnHMznXPVzrnqyspKr0MWEY+srann6PJCSgrCrd6nb7di7r2kmve272X6o0uJJT70t6R4zNNEYWYRUklitnNuTjNFrgDmuJS1wAbgOC9jEhH/rK2tZ3Aru50OdtLAbnx/6kjmr93Krb9f5UFkkomXVz0Z8ACw2jk3o4Vi7wIfT5fvCQwFNIWkSA5KJh3rava0enyiqc9V9+VLpw/iVy+/w8ML3m7f4CSj1rf/2m4CcAmw0syWp7d9E+gH4Jy7B/geMMvMVgIG3Oyc2+phTCLik8279rEvljjsRAHwjTOPY13tHm59ZhUDu5dw+rHqis4GzxKFc24+qR//TGU2A2d4FYOIdByNVzy1ciC7OXkh4+cXjOG8uxcwffZSfjf9VIb0KGuvEKUFujNbRLKirZfGtqSkIMwDl59IQSSPK2ctZvseTUvuNSUKEcmKdbX1dC2OUFFacMTHqjqqiJmXnsD7u/dzza+XEI3rSigvKVGISFbU1jXQq7yo3Y43rl9Xbj9/FK9s2M4tc1ZqDQsPeTmYLSLSKJpw5Ifb92/TKWOq2LB1Dz977i36VxTz1Y8f067HlxQlChHJilg8SX5exutbDsv1Hz+Gd7fvZcazb9K3WxFTx/Zp98/o7JQoRCQr4slks2tQHCkz43/OHcWWnfv5xuMr6F1exPhBFe3+OZ2ZxihEJCuiCUfYg0QBkB8Occ/nT6Bft2K+9KslvPVBnSef01kpUYhIVnjV9XRAeXGEWVecRH44xKUPvsKmnfs8+6zORolCRLIilvCm6+lgfbsV88iVJ1HfEOfSBxbqHot2okQhIlkRTzrPEwXAsN5duP/Sat7bsY8rZi1iT4OWuDlSShQikhXReJKwh11PBzt5UAV3XjiWlRt38qVfLWF/LJGVz81VShQikhWxRJL8LLQoDjhjRC9+eN4o5q/dyrW/XkJDXMnicClRiEhWZGOMoqnPVffl+1NHMndNLdNnL9NUH4dJiUJEsiKeyM4YRVMXndyPW6eM4LnVH3D9b5dphbzDoEQhIlkRTSSJZGmMoqlLTxnAtz41jD+99j43/Ha5kkUb6c5sEckKP7qeDvaFjwzCObjtj6tJJB13XDi23eeeylU6SyLiuUTSkXT4migAvnj6IL796eH8+fX3mf7oUo1ZtJIShYh47kBXTyTsT9fTwa46bSDf/cxwnl31AV+erUtnW0OJQkQ815goQh3jJ+fyCQP53meP57nVNVw5axH1uikvo45RayKS02KJ1KJCfg1mN+eS8f2ZMW00Czds5+L7F7Jzr6b7aIkGs0XEc//seupYf5ueO64PpQVhrvvNMqbd+xJfOG0QKzbtZNm7O3lv+15uOvM4Lhnf3+8wfdexak1EclJjovB5MLs5Z4zoxawrTmTTjn1844kVPLVsM12L8zmuVxe+/eRr/Oy5Nzv9MqtqUYiI5zpi19PBTh3cnbk3TWLX3hiDK0sJhYx4Ism/z1nJz557i231Ub57zgjyQh0zfq8pUYiI5zpyi+KAHmWF9CgrbHwdzgtx+/mjqCjJ595569m+N8qMaaMpCOf5GKU/lChExHMH7lfoyImiOWbGLWcPo6I0n+//8Q1q6xq475JqyosjfoeWVcGqNREJpHgy1fWUzdlj29PVpw/m5xeMYfm7OznvngVs3LHX75CyKpi1JiKBcqDrKVvrUXhhypgqHrnqJGp272fqLxewcuMuv0PKGiUKEfFcLKBdT02NH1TBE9eeSn5eiM/du4A/rtzid0hZ4VmtmVlfM5trZqvM7HUzu76FcpPMbHm6zN+9ikdE/BMNwGB2ax3Ts4wnp09gxNHlfHn2Un7+3Fs5f/msl7UWB77mnBsOjAemm9nwgwuY2VHAL4FznHMjgM95GI+I+CSeCPYYRVOVZQXM/sLJnDu2ip8+9yZf+c2ynF6b27Nac85tcc4tTT+vA1YDVU2KXQTMcc69my5X41U8IuKfXBijaKowksdPpo3m5jOP4w8rt3D2Hf9gyTs7/A7LE1lJ72Y2ABgLLGzy1rFAVzN7wcyWmNmlLex/tZktNrPFtbW13gYrIu0ul7qeDmZmXDtpMP979SnEE47P3bOAGX9dk3MLI3lea2ZWCjwB3OCc293k7TBwAvApYDLwbTM7tukxnHMznXPVzrnqyspKr0MWkXYWy7Gup6ZOGtiNP9/wEaaO7cMdf1vLeXcv4K0P6vwOq914WmtmFiGVJGY75+Y0U2Qj8Bfn3B7n3FZgHjDay5hEJPviHWg9Cq+UFUb4ybTR3H3xON7bvpdP/WI+981bTyIZ/IFuL696MuABYLVzbkYLxZ4CTjOzsJkVAyeTGssQkRzSOEbRQdaj8NJZI3vz13+byMRjK7ntj6u5YOZLvLNtj99hHREva20CcAnwsfTlr8vN7Gwzu8bMrgFwzq0G/gysAF4B7nfOveZhTCLig2iOdz01VVlWwMxLTmDGtNG88X4dk382jwfnbyAZ0NaFZ3M9OefmA4dsZzrnbgdu9yoOEfFfR1oKNVvMjHPH9eHUwd355u9Wcuszq/jTa1v40fmjGdi9xO/w2qRzpHcR8VU8R696ao1e5YU8cFk1P/ncaNa8X8eZAWxddL5aE5GsO9D1FO6k6zmYGeed0Idnb5zIhCHdufWZVVx438u8tz0YkwsqUYiI52KJJJE8I3WNS+fVs0uqdfGj80exavNuJv9sHr966e0O37pQohARz8XiyU7Z7dQcM2NadV/+/G+nc0L/rnz7qdc5/54FrHm/4953oZoTEc/Fk06Joomqo4p45MqTmDFtNG9v28un7vgHP/7LGvbHEn6H9iGqORHxXDTd9ST/6sCVUc/dOJEpY6q4c+5azvjpPOau6VjT3ilRiIjn1PWUWbeSfH4ybTSPfvFkInnGFQ8t4tpfL2Hzzn1+hwYoUYhIFqQGs/VzcyinDu7On64/nZsmD+Vvb9Qw6ccv8J2nXvM9YajmRMRzsaRT11Mr5YdDTP/oEJ67cSJTx1Qxe+G7TLx9LrfMWenb5bRKFCLiOXU9tV3fbsX88PxRvHDTJKZV9+WJJRt56MW3fYnFsyk8REQOUNfT4evTtZjbpo7kuo8N8W2uLCUKEfFcLKGupyPVu7zIt89WihcRz0XVogg01ZyIeC6uRBFoqjkR8Zy6noJNiUJEPKfB7GBTzYmI56KJJJGwfm6CSjUnIp6LJxyRTroWRS5QohARz6nrKdhUcyLiuZi6ngJNNScinovGk77dVSxHTjUnIp6LJ12nXS87FyhRiIjn1PUUbKo5EfGUcy59w51+boJKNScinoolHAD5ujM7sJQoRMRT8WQSgLBaFIGlmhMRT8XiqRaFup6CSzUnIp6KJlItCnU9BZdnicLM+prZXDNbZWavm9n1GcqeaGZxMzvfq3hExB+xdKJQiyK4vFzhLg58zTm31MzKgCVm9qxzbtXBhcwsD/gh8FcPYxERn8TTg9kaowguz2rOObfFObc0/bwOWA1UNVP0K8ATQI1XsYiIf6KNLQp1PQVVVlK8mQ0AxgILm2yvAqYCdx9i/6vNbLGZLa6trfUqTBHxQKxxjEItiqDyvObMrJRUi+EG59zuJm//DLjZOZfMdAzn3EznXLVzrrqystKrUEXEAxqjCL5WjVGY2WBgo3OuwcwmAaOAR5xzOw+xX4RUkpjtnJvTTJFq4LdmBtAdONvM4s65J9vwbxCRDizWOEahrqegam2KfwJImNkQYCbQF3g00w6W+vV/AFjtnJvRXBnn3EDn3ADn3ADgceDLShIiuUVdT8HX2queks65uJlNBX7hnPuFmS07xD4TgEuAlWa2PL3tm0A/AOfcPYcVsYgESmPXkyYFDKzWJoqYmV0IXAZ8Jr0tkmkH59x8oNVtTefc5a0tKyLBoTGK4GttzV0BnALc5pzbYGYDgV95F5aI5IrGMQqtRxFYrWpRpG+S+yqAmXUFypxzP/QyMBHJDY1jFOp6CqxW1ZyZvWBmXcysG7AUuM/Mmh2gFhE5mLqegq+1NVeevgfiXFKXxZ4MfMK7sEQkV/xz9lh1PQVVaxNF2Mx6A9OAZzyMR0RyTCypFkXQtbbmbgX+Aqxzzi0ys0HAW96FJSK5IhZXogi61g5mPwY8dtDr9cB5XgUlIrnjwFVP6noKrtYOZvcxs9+ZWU368YSZ9fE6OBEJvqgGswOvtTX3EPA0cHT68fv0NhGRjOIJLYUadK2tuUrn3EPOuXj6MQvQNK4ickixRJKQQZ5uuAus1iaKbWb2eTPLSz8+D2zzMjARyQ2xRFKtiYBr7VxPVwK/AH4KOGABcLlHMXliyTvbmTlvPeFQiFDIGqcTiCaSxOJJYokkSQdJ50g6RyLpSCYhnkySSDrMjPxwiIJwiHDIaIgn2RdLsC+aSP/FZITMMIP0tOktMkiXI7UPqX3yQkYkz8gP55GfF6Igkvq8wkjqdePxSS0rWRTJozg/j8L8PMIhO+i4qX9fOC/UuD1x4N/kHCEzInkh8tLvRRNJGmJJookkSecwjFA6vqSDRNLhXOocFOfnUZwfpjg/j7LCMOVFEcqLIpQVRnTnrTQrmkhq5tiAa+1VT+8A5xy8zcxuILXwUCDUNyR4Z9te4snUD2Yi6XA48vNCRNKPUCj1A5mX/tHPCxkFkTAhM5LOEY0nqW+IE084CsIhSgvCVJYWEMkL4UglloRzGeNIve1wLpVxnXMk088TySTReJJd+2JE40ka4gkaYkka0s+dS5V3pPp9DwwSdhR5IaMokkdhJERxfpjSgjClhWHKCsJ0LcmnoiSfbiX5dC3Op6QgTElBHqUFYcoKI3QpSiWdokjeIROtBEs84TRzbMC1tkXRnBsJUKKYeGwlE4/NrWGVeCLdqokl0n/1pxJOMp0IDyTFpHOEQ5ZOhNaYKOPJJM7R2FLKD6daLanjpBJYKmlCKJTab280wd5onL3RBHX7Y+zaF2PX3hh1++PsiyXYH0uyP55qadXtj1PfEGPLrv2s2rKbbXuiROOZk1t+XojupflUdimkR1kB3YrzKcpPtZyKInkclU44FSX5VJQW0L00ny6FEULq/+6wYomkJgQMuCNJFKp5n4XzQpTlhSgrzDjje4fhnGNPNMGOPVH2ROPsaYhT35Cgfn+c3emks3NvjNq6Bmrq9vPe9r28+t7OdAJKNF6P31Q4ZHQryadnl0L6diuib9di+nQrZlD3EgZ2L6FXl0IlEh9FNUYReEeSKDL3sYg0YWap7qiCw/vaReNJdu6Lsq0+yvY9UbbWN7CtPsq2PQ1srYuyZfd+3thSx3Orav6lW64okseA7iUMqChmQPcSBlaUcEzPUob17kJhJK+9/nnSgljCafwq4DL+jzWzOppPCAYUeRKRSAvywyF6lBXSo6wwY7lk0vFB3X42bN3D+trUY8PWeta8X8ezqz4gnkx9pfNCxpDKUkZUdeHEAd04aWA3BnUv0RhJO4snkrorO+AyJgrnXFm2AhFpL6GQ0bu8iN7lRZw6uPu/vBdPJNm0cx+rt9SxavMuXtu8m3lv1jJn6SYAKssKmDC4grNH9ub0YyvV4mgHqTEKtSiC7Ei6nkQCJ5wXon9FCf0rSjjz+F5Aauxk/dY9LFy/nYUbtvHCm7U8uXwzZQVhPjm8J1PGVnHakO66YewwRXXVU+ApUUinZ2YMrixlcGUpF53cj1giyYJ12/jDis38+bX3mbNsEz27FDB1bB/OP6EPQ3qU+h1yoMTiSfLV9RRoShQiTUTyQo2XU3/vs8fzt9U1PL5kI/f9Yz33zlvHuWP7cNPkofQqzzxWIinxpK56CjolCpEMCsJ5nDWyN2eN7E1N3X4emL+Bh+a/zR9Wbubq0wfzpdMHUXKYV3F1FtGEoyhfiSLIVHsirdSjrJBbzhrG81+byMeH9eSO59/ijJ/O4+9v1vodWoemrqfgU6IQaaO+3Yq566JxPHbNKRRGQlz24Ct84/FX2bUv5ndoHZImBQw+1Z7IYTpxQDf+8NWPcO2kwTyxdBNn/PTvLH13h99hdTjxpFOiCDjVnsgRKIzkcfOZx/HklydQGMnj8/cvZMG6rX6H1aFE40nC6noKNCUKkXYwsk85j33pFPp0LeKKhxYx940av0PqMGKaZjzwVHsi7aRHl0J+e/UpHNOzlKt/tZg/rtzid0gdgsYogs+z2jOzvmY218xWmdnrZnZ9M2UuNrMVZrbSzBaY2Wiv4hHJhm4l+Tz6xfGM6nMU1z26lMcWv+d3SL6LJzRGEXRe1l4c+JpzbjgwHphuZsOblNkATHTOjQS+B8z0MB6RrOhSGOFXV53EhCHduenxFTz04ga/Q/JVVJMCBp5nicI5t8U5tzT9vA5YDVQ1KbPAOXfgMpGXgT5exSOSTcX5Ye6/rJrJI3ryX79fxR3Pv4U7xOqHuUpdT8GXldozswHAWGBhhmJXAX9qYf+rzWyxmS2urdXNTRIMBeE87rpoHOeOq2LGs2/y7adeO+QKf7kmtcIiShQB5/ncA2ZWCjwB3OCc291CmY+SShSnNfe+c24m6W6p6urqzvlnmQRSOC/Ej88fTWVpAffOW8+qzbv55cUndJp5omLpBaQiYXU9BZmnad7MIqSSxGzn3JwWyowC7gemOOe2eRmPiB9CIeOWs4dx10XjeOP9Oj79i3/w0rrO8VVvTBRajyLQvLzqyYAHgNXOuRktlOkHzAEucc696VUsIh3Bp0b15unrJlBeFOHzDyxk0dvb/Q7JcwfWOddgdrB5meYnAJcAHzOz5enH2WZ2jZldky7zn0AF8Mv0+4s9jEfEd0N6lPHk9An06lLIf/xuZc6PWfyz60ktiiDzbIzCOTef1Nramcp8AfiCVzGIdERlhRFunTKCqx5ezAPzN3DtpMF+h+SZxkShwexAU+2J+ODjw3oyeURPfv78m7y3fa/f4XhGXU+5QYlCxCff+cwIQmb851Ov5ew9FmpR5AbVnohPjj6qiBs/eSxz19Tyl9ff9zscTxwYg1GiCDbVnoiPLj91AMN6d+HW36+iIZ7wO5x2F0+mWkqaPTbYVHsiPgrnhfjWp4axedd+Hl34rt/htLsDXU9ajyLYlChEfDZhSHdOHVzBXXPXsqch7nc47SqmrqecoNoT6QC+PnkoW+ujzFrwtt+htKuoBrNzgmpPpAMY168rnxjWk3v+vo5de2N+h9Nu4gmNUeQC1Z5IB/H1ycdS3xDn3nnr/A6l3WiMIjcoUYh0EMf16sKU0Ufz0ItvU1O33+9w2oW6nnKDak+kA/m3Tx5LLJHkl3Nzo1URU9dTTlDtiXQg/StKOG9cH37zyrvU7A5+qyKu9ShyghKFSAcz/aNDiCcdd/89+K2KxjEKrUcRaKo9kQ6mX0Ux542r4tGFwW9VRNX1lBNUeyId0HUfPYZ40nHP39f7HcoR0VKouUGJQqQD6ldRzLljq5i98J1Atyp0Z3ZuUO2JdFDXfWxI4FsVsfSkgOGQWhRBpkQh0kH1ryhpbFWseb/O73AOSyyRJJJnmClRBJkShUgH9vXJQykvinDVw4vYVt/gdzhtFosn1e2UA1SDIh1Yzy6FzLy0mpq6Bq6dvbRxIaCgSLUo9DMTdKpBkQ5uTN+juP38UbyyYXvglk2NJZ3Wy84BYb8DEJFDmzKmijc/qOOuuesY1rsLl506wO+QWkVdT7lBNSgSEF/75FA+dlwPvv/H1byzbY/f4bSKup5yg2pQJCBCIeP7U0cSyQvxH78LRhdULKGup1ygRCESIL3KC7n5zKHMX7uV3y3b5Hc4h6QWRW5QDYoEzMUn9+eE/l353jOrOvwls0oUuUE1KBIwoZDxg3NHUt8Q57Y/rPY7nIzU9ZQblChEAujYnmVcO3Ewc5Zt4uX12/wOp0VRtShygmc1aGZ9zWyuma0ys9fN7PpmypiZ3WFma81shZmN8yoekVzz5Y8OoXtpPvd24HUr4koUOcHLGowDX3PODQfGA9PNbHiTMmcBx6QfVwN3exiPSE4pjOTx+fH9mbumlrU19X6H0yx1PeUGzxKFc26Lc25p+nkdsBqoalJsCvCIS3kZOMrMensVk0iu+fz4/uSHQzz44ga/Q2mWBrNzQ1Zq0MwGAGOBhU3eqgLeO+j1Rj6cTESkBd1LC5g6poo5SzeyY0/U73A+JJpIEgkrUQSd5zVoZqXAE8ANzrndh3mMq81ssZktrq2tbd8ARQLuytMGsj+W5NFX3vU7lA+JJxwRrUUReJ4mCjOLkEoSs51zc5opsgnoe9DrPult/8I5N9M5V+2cq66srPQmWJGAGtqrjI8c052HF7zd4WaXVddTbvDyqicDHgBWO+dmtFDsaeDS9NVP44FdzrktXsUkkquuPG0gNXUNPLNis9+h/IuYup5ygpezx04ALgFWmtny9LZvAv0AnHP3AH8EzgbWAnuBKzyMRyRnTTymkiE9Srn/HxuYOraqw6woF40nyVeLIvA8SxTOuflAxm+rS81qNt2rGEQ6i1DIuGbiYL7+2Ks8/epmpozpGNeExJNO62XnAKV6kRxx7tgqRhzdhR/+6Q32xxJ+hwOo6ylXqAZFckQoZHz708PZvGs/9/9jvd/h4JxL33Cnn5mgUw2K5JDxgyqYPKInv3xhHTW79/saSyyRWi8jX3dmB54ShUiOueWsYcQSSX781zW+xhFPpi7VDatFEXiqQZEcM6B7CZedMoDHlmzktU27fIsjFk+1KNT1FHyqQZEc9JWPH0O34nwue/AV5r+11ZcYoolUi0JdT8GnRCGSg8qLIvzvl06hojSfSx5cyB3Pv0Uymd01tmPpRKEWRfCpBkVy1JAepTw5fQKfHVPFjGff5IpZi6jbH8va58fTg9kaowg+1aBIDivODzNj2mhum3o8L67dyuUPLaK+IZ6Vz442tijU9RR0ShQiOc7MuPjk/tx50ViWv7eTyx98JSvJItY4RqGfmaBTDYp0Emce35tfXDiWZe/t5MqHFrHH42ShMYrcoRoU6UTOHtmbOy4Yy5J3d3DVw4toiHs31UescYxCXU9Bp0Qh0sl8alRvfvK50by8fjvfeHyFZ1dDqespd3g5zbiIdFCfHVvFpp37uP0va+jTtYibJh/X7p/R2PWkSQEDT4lCpJP68qTBbNyxj7vmrqPqqGIuOrlfux5fYxS5Q4lCpJMyM743ZQRbdu3j20+9Ru+jCvno0B7tdvwtu1KTEpYXRdrtmOIPpXqRTiycF+LOi8ZxXK8yps9e2q5zQ/19TS1VRxUxoKK43Y4p/lCiEOnkSgvCPHT5iXQtzufyhxbx3va9R3zMaDzJi2u3MnFoZYdZllUOnxKFiNCjSyGzrjiRaDzBZQ+9ws690SM63uK3t7MnmmjXrizxjxKFiABwTM8y7ru0mo3b9/HFRxY3DkYfjhferCU/L8SpgyvaMULxixKFiDQ6eVAFPzp/FIve3sHdL6w77OPMfaOGkwZ2o6RA18vkAiUKEfkXnx1bxTmjj+aO59/i9c1tH9zetHMfb9XUM2lopQfRiR+UKETkQ/7rnBEcVZzP1x9bQTTeti6oF9bUADBJ4xM5Q4lCRD6ka0k+3596PKu37OauuWvbtO/cN2rp07WIwZUlHkUn2aZEISLNOmNEL84dW8Vdc9e2+v6KhniCBeu28tGhPXRZbA5RohCRFn3nMyOoKM1n2r0vMeOvaw65Qt6iDTvYG01ofCLHKFGISIvKiyM89qVT+dhxPbjjb2uZePsLPDh/A/tjzU9P/sKaGvLDIU7RZbE5RQPgDn4AAAdXSURBVIlCRDLqV1HMnReN4+nrJnBcrzJufWYVp/3wb/zi+bfYsSd1Y96WXft4cP4Gnly+iZMHdqM4X5fF5hJzzpu56L1SXV3tFi9e7HcYIp2Sc46X129n5rx1zF1TS1EkjyE9SlmZHsM4rlcZt009nhP6d/M5UmnKzJY456oPZ1/P0r6ZPQh8Gqhxzh3fzPvlwK+Bfuk4fuyce8ireETkyJkZpwyu4JTBFbz5QR33zVvP+q17uGnyUM46vheDKkv9DlE84FmLwsxOB+qBR1pIFN8Eyp1zN5tZJbAG6OWcyzjJjFoUIiJtdyQtCs/GKJxz84DtmYoAZZa6hq40Xdbb1d5FRKTN/BzMvhMYBmwGVgLXO+eavQXUzK42s8Vmtri2tjabMYqIdHp+JorJwHLgaGAMcKeZdWmuoHNupnOu2jlXXVmp67NFRLLJz0RxBTDHpawFNgDtv8K7iIgcET8TxbvAxwHMrCcwFFjvYzwiItIMLy+P/Q0wCehuZhuB7wARAOfcPcD3gFlmthIw4Gbn3Fav4hERkcPjWaJwzl14iPc3A2d49fkiItI+NIWHiIhkFLgpPMysFnjnMHcvB1q7ZFdrymYq09J7rd3eXLnuQLa659pyro50/2ye69Zsy9Xz3Jryh/u+znXby2b7O13inDu8y0adc53mAcxsz7KZyrT0Xmu3N1cOWNwRz1WQznVrtuXqeW5N+cN9X+e67WWD9J3ubF1Pv2/nspnKtPRea7e3JVYvHOnnd9Rz3dpt2ZLN89ya8of7vs5128sG5jsduK6nzszMFrvDnKtFWk/nOXt0rrPjSM9zZ2tRBN1MvwPoJHSes0fnOjuO6DyrRSEiIhmpRSEiIhkpUYiISEZKFCIikpESRQ4ws8+a2X1m9r9mpmlRPGRmg8zsATN73O9Yco2ZlZjZw+nv8sV+x5PL2vo9VqLwmZk9aGY1ZvZak+1nmtkaM1trZv+e6RjOuSedc18ErgH+n5fxBlk7nev1zrmrvI00d7TxnJ8LPJ7+Lp+T9WADri3nuq3fYyUK/80Czjx4g5nlAXcBZwHDgQvNbLiZjTSzZ5o8ehy067fS+0nzZtF+51paZxatPOdAH+C9dLFEFmPMFbNo/bluE89mj5XWcc7NM7MBTTafBKx1zq0HMLPfAlOccz8APt30GOl1x/8H+JNzbqm3EQdXe5xraZu2nHNgI6lksRz9EdtmbTzXq9pybFVGx1TFP/+ygtR/oKoM5b8CfAI438yu8TKwHNSmc21mFWZ2DzDWzG7xOrgc1dI5nwOcZ2Z34/8UNrmi2XPd1u+xWhQ5wDl3B3CH33F0Bs65baTGgqSdOef2kFoiWTzW1u+xWhQd0yag70Gv+6S3SfvTuc4+nfPsaZdzrUTRMS0CjjGzgWaWD1wAPO1zTLlK5zr7dM6zp13OtRKFz9Jri78EDDWzjWZ2lXMuDlwH/AVYDfyfc+51P+PMBTrX2adznj1enmtNCigiIhmpRSEiIhkpUYiISEZKFCIikpEShYiIZKREISIiGSlRiIhIRkoUkjPMrD7Ln3f/4czEeYSfeYOZFWfzM0V0H4XkDDOrd86VtuPxwukblrImPROwOeeSLbz/NlDtnNuazbikc1OLQnKamVWa2RNmtij9mJDefpKZvWRmy8xsgZkNTW+/3MyeNrO/Ac+b2SQze8HMHjezN8xsdvrHnPT26vTzejO7zcxeNbOXzaxnevvg9OuVZvbfzbV6zGxAemGZR4DXgL5mdreZLTaz183sv9LlvgocDcw1s7npbWek/x1LzewxM2u3RCnSyDmnhx458QDqm9n2KHBa+nk/YHX6eRcgnH7+CeCJ9PPLSU3F3C39ehKwi9RkaiFSUyQcON4LpP66B3DAZ9LPfwR8K/38GeDC9PNrWohxAJAExh+07cDn56U/Z1T69dtA9/Tz7sA8oCT9+mbgP/2uBz1y76FpxiXXfQIYnm4EAHRJ/9VdDjxsZseQ+pGPHLTPs8657Qe9fsU5txHAzJaT+mGf3+RzoqSSAsAS4JPp56cAn00/fxT4cQtxvuOce/mg19PM7GpSSwH0JrU62Yom+4xPb38x/e/LJ5XIRNqVEoXkuhCpv9T3H7zRzO4E5jrnpqZXBXvhoLf3NDlGw0HPEzT//ybmnHOHKJNJ42ea2UDg68CJzrkdZjYLKGxmHyOV1C5s42eJtInGKCTX/ZXUCoAAmNmY9NNy/jkv/+Uefv7LwHnp5xe0cp8upBLHrvRYx1kHvVcHlB107AlmNgTAzErM7NgjD1nkXylRSC4pTk+vfOBxI/BVoNrMVpjZKv65qtePgB+Y2TK8bVnfANxoZiuAIaTGOzJyzr0KLAPeINVd9eJBb88E/mxmc51ztaSS3G/Sx38JOK59wxfR5bEinkrf87DPOefM7AJSA9tT/I5LpC00RiHirROAO9OX1O4ErvQ5HpE2U4tCREQy0hiFiIhkpEQhIiIZKVGIiEhGShQiIpKREoWIiGSkRCEiIhn9f5Ht8Rp9TVdoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "reset_seed(SEED)\n",
    "trainloader = DataLoader(trainset, batch_size=DESIRED_BATCH_SIZE, shuffle=True)\n",
    "\n",
    "device = torch.device('cuda')\n",
    "model = ConvNet()\n",
    "model = model.to(device)\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.5)\n",
    "criterion = nn.NLLLoss()\n",
    "\n",
    "lr_finder = LRFinder(model, optimizer, criterion, device='cuda')\n",
    "lr_finder.range_test(trainloader, end_lr=10, num_iter=100, step_mode='exp')\n",
    "lr_finder.plot()\n",
    "lr_finder.reset()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training loss and gradient accumulation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Desired `batch_size`: 256\n",
    "- Real `batch_size`: 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "efcfd22e97474c55b03e59caa1cb0fd6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping early, the loss has diverged\n",
      "Learning rate search finished. See the graph with {finder_name}.plot()\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd3hc1Z3/8fd3ZtSLZVvFtlzkgovccAEbDITeewiEGlgSwoaE8EtZQrJJNsmmL9knLGQTkxAgAbIxmB46BAKmyb2DK7aKJTdJltVm5vz+0JgYIwu1O3ek+byeZ55nNHPm3q+vLH107rn3HHPOISIiySvgdwEiIuIvBYGISJJTEIiIJDkFgYhIklMQiIgkOQWBiEiSC/ldQFfl5+e7kpISv8sQEelTFi9evNM5V9Dee30uCEpKSigrK/O7DBGRPsXMth7uPZ0aEhFJcgoCEZEkpyAQEUlyCgIRkSSnIBARSXIKAhGRJNfnLh/1QmskSm1jK6GAkZueQiBgH3m/sSVCfVMr6alBslJDBAOGc4765jDVdU3sqGsGoCAnjYLsNPIyUzCz9nYlIpJwkiYIFm3YyX89v55w1BGOOMLRKI2tEfY2tFLfHP6wXTBgDMxMIS8zlcaWCLsbWmhsjXxkWxkpQYCPvX5AStAoyk1nWF4GxXkZFOamETokXIJmBAJG0Iy0lABZaSGyY48BGSnkZaYwICOV3IwQqcGAgkVEPJM0QRAMGFlpIUIBIxgIkBI00kIB8jJTGZiZSl5mCuGoY3dDM7sbWtm7v4WM1CCDMlMZlJ1KbnoKTa0R9jWHaWgO4xwU5aZTmJtGUW46BlTXN1NT30x1fTNVtY1U7G3i3S27qa5rJnrQAkAOiES7tiBQajBAaqit7pRggJTY17npIYYPzGT4wAyKB2YwKCuVnPQUctLbAqUoN53stKT5NotIN3j2G8LMRgD3A0W0/e6b75z79SFtLgB+BESBMHCLc+51L+qZM2Ywc8YM9mLT3RaNOiLO0RyO0tAcpr4pzL7mMHWNrextbKV2fwu1ja20hKO0RBytkSgt4SjhaJTmcNvz2sZW1lTW8cKaHbREou3uJyctxNC8dAZlpWIYUedwtIVLdlqI7PS2nkh6SpCUoBEKBEgJ2Yc9lAO9lYzUIOmhIBmpQbLTQgzMTCEU1DCTSF/n5Z+KYeDrzrklZpYDLDazF5xzaw5q8xLwhHPOmdk04K/ARA9rSiiBgBGg7S/87LQQRbnd31Y06ti5r5k9+1upb2o73VXX2EpVbROVtU1U1jayu6EFcBgGBg0tYarrm9jXFKa+OUxzOEprJEpnVy81gwEZKQzKausx5aSHyEoNkZMeYnRBFhOKcpgwJIfivAyd2hJJYJ4FgXOuEqiMPa83s7VAMbDmoDb7DvpIFm09B+mGQMAozE2nMDe9x9uKRB0t4SgNLWH2xXop+5rDNLVGaGqN0Ngaoa4xzO6Glg8fdU2t7GsOs6Ouib37W1mwePuH28tOCzG2MJtxBdmMK8xmTEEWowZnMnJQJpmpOm0l4re4/BSaWQkwA3i7nfcuAn4KFALnHObzNwA3AIwcOdKrMiUmGDAyUttOAeVnp3VrG3VNrbxXVc+6qnre21HPhup9/OP9Gh5Zsv0j7fKz0xgyII1BWWkMzkplUFYq+dlp5Genkp+TRlFOOmMLs0gLBXvjnyYi7TDX2fMA3d2BWTbwKvBj59zCDtqdAHzPOXdqR9ubPXu20+yjfVdtYytbdjbwwe79bY9d+6mub2J3Qwu7Yr2L/S0fvRorFDDGFmRTOiyXCUNyKBmcxej8tl5FeooCQqQzzGyxc252e+952iMwsxTgEeCBjkIAwDn3mpmNMbN859xOL+sS/wzISGH6iDymj8g7bJv9LWF21rdQs6+Zir2NrKuqY01FHW9u3MWjS8s/0vaIwmyOHTuYY8bmM3fMIPIyU73+J4j0O571CKxtdPA+YLdz7pbDtBkHbIwNFs8EngSGuw6KUo8gudU1tbJ1534272pgc00DSz7Yw7tbdrO/JYIZjC/MYeaogcwaNZCjSgYyanCW3yWLJAS/egTzgKuBlWa2LPbat4GRAM653wKfBq4xs1agEbisoxAQyU1PYerwAUwdPuDD11rCUVZs38ubG3dRtnUPT62o4KF3PgBg4pAczpk6lHOmDWVMQbZfZYskNM/HCHqbegTySaJRx4aafbyxYSdPr6ikbOseAMYVZjNzZB5HjhjIjJF5jC/KIRjQZa2SHDrqESgIpN+rrG3k6RWVvL5hJ8u37WXP/lYABmelclppEWdMHsKx4wbryiTp1xQEIjHOObbu2s/SbXt4ZV0NL6+rZl9zmOy0EBfNKOa6eSU6hST9koJA5DCawxEWbdzFk8sreGp5Ja3RKKdMLOT648ZwzNjEmpJEpCcUBCKdUFPfzJ/e2sqf39rK7oYWThhfwHfOnsSEITl+lybSYwoCkS5oao3w57e2csdL77OvOcxnjx7J104b3+27rEUSQUdBoKkjRQ6RnhLk88eP4dVvnsQ1x5Tw13e3ceqvXuUf79f4XZqIJxQEIocxMCuV/zh/Ms/ecjxFOel87p53+N2rG+lrvWiRT6IgEPkE4wpzWPilYzlzyhB++sw6bv7LMva3hD/5gyJ9hIJApBOy0kLcdcVM/u3MCTy1ooKrfv+2wkD6DQWBSCeZGV86cRx3XTGTZdv28qUHltB6mFXhRPoSBYFIF509dSj/eeFU/r6+hn97eAXRLq4/LZJotDyUSDdcMWcku/Y1c/sL7zE4K5XvnDNJy3FKn6UgEOmmL588jl0NLfz+9c0MH5jBtfNG+12SSLfo1JBIN5kZ3zu3lFMnFfHjv61l2ba9fpck0i0KApEeCASM2z8znaLcdG56YAl797f4XZJIlykIRHpoQGYKd10xk+r6Jr6xYLluOJM+R0Eg0gumj8jjO2dP4sW11dz9j01+lyPSJQoCkV7yuWNLOHvqEH7+7HqWfrDH73JEOk1BINJLzIyffXoaRTlpfH3BcppaI36XJNIpCgKRXpSbnsIvPzOdTTUN/OLZ9X6XI9IpCgKRXjZvXD6fO2YU97yxmTc37vK7HJFPpCAQ8cCtZ02kZHAm33x4OfuaNTmdJDYFgYgHMlND3H7pdCr2NvLjp9f4XY5IhxQEIh6ZNWoQXzh+DA+9s40luopIEpiCQMRDN59yBIU5afzwyTWapVQSloJAxENZaSH+7cyJLNu2lyeWV/hdjki7FAQiHrt4RjFTiwfws2fWaVUzSUieBYGZjTCzV8xsjZmtNrOvttPmSjNbYWYrzWyRmU33qh4RvwQCxvfOK6Wqron5r2n6CUk8XvYIwsDXnXOlwFzgJjMrPaTNZuBTzrmpwI+A+R7WI+Kbo0oGcc60ofz21Y1U1jb6XY7IR3gWBM65SufcktjzemAtUHxIm0XOuQOXU7wFDPeqHhG/fevMiUQduuNYEk5cxgjMrASYAbzdQbPrgWcO8/kbzKzMzMpqamp6v0CROBgxKJPrjxvNY8vKWV1R63c5Ih/yPAjMLBt4BLjFOVd3mDYn0RYEt7b3vnNuvnNutnNudkFBgXfFinjsxk+NJTc9Rb0CSSieBoGZpdAWAg845xYeps004PfABc45Tcwi/dqAjBRuOmksr75Xw6KNO/0uRwTw9qohA/4ArHXO/eowbUYCC4GrnXPveVWLSCK55pgShg1I5+fPrtdqZpIQvOwRzAOuBk42s2Wxx9lmdqOZ3Rhr8z1gMPCb2PtlHtYjkhDSU4Lcctp4lm/by7OrqvwuRwTra3+RzJ4925WVKS+kb4tEHWf9+jXCEcfz/+8EQkHd2yneMrPFzrnZ7b2n/30iPggGjG+eMZFNOxtYsHi73+VIklMQiPjk1EmFzBiZx50vb6AlHPW7HEliCgIRn5gZN59yBOV7G3l0qXoF4h8FgYiPThxfwPThA7jzlQ20RtQrkMO78U+LecSj04gKAhEfHegVbNvdyGNLy/0uRxLUuqo6nl1d5dmypwoCEZ+dPLGQKcW53PnKBsLqFUg7nlhWQTBgnD11qCfbVxCI+MzMuPnkI9i6a78Wr5GPcc7xxPIK5o3LpyAnzZN9KAhEEsBppUVMGprLnS9vIKIlLeUgSz7Yy/Y9jVwwfZhn+1AQiCSAtl7BODbtbODv66v9LkcSyBPLykkLBTh9cpFn+1AQiCSIUyYVkZMe0rQT8qFwJMrTKys5ZVIhOekpnu1HQSCSIFJDAU6ZWMiLa3do0FgAWLRxFzv3tXD+9OJPbtwDCgKRBHLG5CHs2d/KO1t2+12KJIDHl1WQkx7ixAnersOiIBBJIJ+aUEBaKMDzq3f4XYr4rKk1wnOrqzhryhDSU4Ke7ktBIJJAMlNDnDC+gOdXV2mtgiT38rpq9jWHPT8tBAoCkYRzxuQhVNQ2sbJc6xonq737W/jtqxvJz07jmLGDPd+fgkAkwZw6qZBgwHT1UJLaVLOPi36ziHWV9fzg/MkEA+b5PhUEIgkmLzOVOaMH8dxqBUGyWbRhJxf9ZhG1ja08+IU5nDPNmyklDqUgEElAZ0wewsaaBjZU7/O7FImThUu2c80971CYk8bjN81jdsmguO1bQSCSgA7cRapeQXJ46J0P+PqC5Rw9ehCPfOlYRgzKjOv+FQQiCWjogAymj8jjeQVBv3ffoi3ctnAlnxpfwD3XHkWuh3cQH46CQCRBnV5axPLttVTXN/ldinjk7tc28f0nVnNaaRG/u3qW5/cLHI6CQCRBnXBE292kb27c5XMl4oX5r23kx39byznThvKbK2eSFvInBEBBIJKwSoflkpeZwuvv7/S7FOll97y+mZ/8bR3nThvKry87kpSgv7+KFQQiCSoYMI4dO5g3NuzUXcb9yJ/e2soPn1rDGZOL+O/LjiTkcwiAgkAkoc0bl09FbRObdzb4XYr0gv979wO++9gqTplYyP9cPtP3nsABiVGFiLTruHH5ALyxQaeH+jLnHL99dSO3PrKSE8YX8JurZpIaSpxfv55VYmYjzOwVM1tjZqvN7KvttJloZm+aWbOZfcOrWkT6qpGDMhk+MIPXFQR9ViTq+O7jq/jZM+s4Z9pQ5l89y9eB4faEPNx2GPi6c26JmeUAi83sBefcmoPa7AZuBi70sA6RPsvMOG5cPk+vrCQSdXGZd0Z6z/6WMDc/tJQX11bzxRPGcOuZEwkk4PfQsx6Bc67SObck9rweWAsUH9Km2jn3LtDqVR0ifd28cfnUN4U1G2kfU9fUylW/f5uX1lXzwwsmc9vZkxIyBCBOYwRmVgLMAN6Ox/5E+pNjY9MQa5yg76hrauWaP7zDiu213HXFTK45psTvkjrkeRCYWTbwCHCLc66um9u4wczKzKyspqamdwsUSXCDs9MoHZqrIOgjDoTAqvJa7rxiJmdPjc8Moj3haRCYWQptIfCAc25hd7fjnJvvnJvtnJtdUODt2p0iiei4I/Ip27KHxpaI36VIB2obW7n6D++wuqKW31w5kzOnDPG7pE7x8qohA/4ArHXO/cqr/Ygkg3nj8mmJRCnbqkXtE1VDc5jr/vgOaypq+c2Vszh9ct8IAfD2qqF5wNXASjNbFnvt28BIAOfcb81sCFAG5AJRM7sFKO3uKSSR/uqokoGkBgO8vmEnxx+hXnGiaWqN8IX7y1i+vZa7rpjBaaVFfpfUJZ4FgXPudaDDIXLnXBUw3KsaRPqLzNQQM0fl8fLaar515kTaOtySCFojUb784BIWbdzFry6dzplTEn9M4FCJc2ubiHTo/OnFvF+9T5eRJpBI1PG1vy7nxbXV/OjCKVw8s2/+XasgEOkjzpk2lLRQgIcXb/e7FKFt2ogfPrmaJ5dX8K2zJnL13FF+l9RtCgKRPmJARgpnTB7C48sqaA7r6iG/3f2PTdz35la+cPxobvzUWL/L6REFgUgfcsms4dQ2tvLimmq/S0lqTyyv4Cd/a5s76LazJvldTo8pCET6kHnj8hmSm87Di7f5XUrSemvTLr7x1+UcXTKI2z8zPWGnjegKBYFIHxIMGBfPLObV92qortNaxvG2oXofN9xfxsjBmcy/xr81hnubgkCkj/n0rOFEHTy6tNzvUpLK7oYW/uXed0kNBfjjtUeRl5nqd0m9RkEg0seMLchm5sg8Hl68XUtYxklzOMIN95exo66J+dfMZsSgTL9L6lUKApE+6JJZI3i/eh8rtuueAq8557j14RWUbd3D7ZdOZ+bIgX6X1Os6FQRmNtbM0mLPTzSzm80sz9vSRORwzp0+lIyUIL964T2iUfUKvHTHSxt4bFkF3zxjAudOG+Z3OZ7obI/gESBiZuOA+cAI4EHPqhKRDuWmp/Dtcybx6ns1/P71TX6X0289u6qK/37xPS6eWcyXTuzb9wp0pLNBEHXOhYGLgP9xzn0T6HsTaoj0I1fNGclZU4bwi2fXs/SDPX6X0++sq6rja39dxpEj8vjJRVP79fxOnQ2CVjO7HPgc8FTstRRvShKRzjAzfnbxNIpy0/nKQ0upbdSKr71lT0MLX7i/jOy0EL+7uv9cJno4nQ2C64BjgB875zab2WjgT96VJSKdMSAzhTsun0FlbRPfXrhSVxH1gnAkypcfWsKO2mZ+d/UsinLT/S7Jc50KAufcGufczc65h8xsIJDjnPu5x7WJSCfMGjWQb5w+gadXVmpCul7w82fX8caGXfznRVOY0Q+vEGpPZ68a+ruZ5ZrZIGAJcLeZadUxkQTxxRPGcHTJIH701Bp26I7jbnt2VSV3/2Mz1xwziktnj/C7nLjp7KmhAbFVwy4G7nfOzQFO9a4sEemKQMD4+SXTaA5H+c6jOkXUHZt3NvDNBSuYPiKP75zT9yeS64rOBkHIzIYCl/LPwWIRSSCj87P45hkTeHFtNU8sr/C7nD6lqTXCv/55McGgcdcVM0gL9e/B4UN1Ngh+CDwHbHTOvWtmY4D3vStLRLrjunmjmTEyj+8/sZqa+ma/y+kzvvvYKtbvqOe/LzuS4QP71/QRndHZweIFzrlpzrl/jX29yTn3aW9LE5GuCgaMX14yjf0tEb73+Cq/y+kTFpRtY8Hi7XzlpHGcNKHQ73J80dnB4uFm9qiZVccej5hZ31ycU6SfG1eYw5dPGsczq6pYX1XvdzkJbUN1Pd97fDVzxwziq6eO97sc33T21NAfgSeAYbHHk7HXRCQBXTFnJKGAsXCpLic9nKbWCF9+cCmZqUF+/dkZBPvBAjPd1dkgKHDO/dE5F4497gUKPKxLRHogPzuNEycU8NjSciKalK5dP3pqDeuq6rn90ulJcdNYRzobBLvM7CozC8YeVwG7vCxMRHrmohnD2VHXzJsb9aN6qKdXVPLA2x/wxRPGcGKSjgscrLNB8C+0XTpaBVQClwDXelSTiPSCUyYVkpMeYuESnR46WPneRr61cAVHjsjjG2dM8LuchNDZq4a2OufOd84VOOcKnXMXArpqSCSBpacEOXfaMJ5ZVUVDc9jvchJCNOr4+l+XEY067vjsDFKCWpsLerZC2dd6rQoR8cTFM4tpbI3w3Ooqv0tJCH94fTNvbdrN98+fzMjByXe/wOH0JAg6HGI3sxFm9oqZrTGz1Wb21XbamJndYWYbzGyFmc3sQT0icojZowYyYlAGC5dooft1VXX88rn1nF5axGdm6er3g/UkCD7pUoQw8HXnXCkwF7jJzEoPaXMWcETscQPwvz2oR0QOYWZcNGM4b2zcSVVt8k5G1xyOcMtflpGbkcJPL+7fi8x0R4dBYGb1ZlbXzqOetvsJDss5V+mcWxJ7Xg+sBYoPaXYBbZPYOefcW0BebE4jEeklF88oxjl4bFny9gp+9cJ7rKuq5xeXTGVwdprf5SScDoPAOZfjnMtt55HjnAt1didmVgLMAN4+5K1iYNtBX2/n42EhIj1Qkp/FrFEDWVC2LSlnJV25vZa7X9vE5UeP4OSJRX6Xk5A8HzI3s2zgEeCW2FTW3dnGDWZWZmZlNTU1vVugSBK4cs5INtY08I/3d/pdSlyFI1Fue3QFg7PT+NZZyTW1dFd4GgRmlkJbCDzgnFvYTpNy4ODVH4bHXvsI59x859xs59zsggLd0CzSVedMG0p+dhp/fGOz36XE1b2LtrCqvI7/OG8yAzK0zPrheBYE1jYa8wdgrXPucKuZPQFcE7t6aC5Q65yr9KomkWSVFgpy1dyRvLK+hs07G/wuJy6279nP7c+/xykTCzl76hC/y0loXvYI5gFXAyeb2bLY42wzu9HMboy1+RuwCdgA3A18ycN6RJLaFXNGkhI07lu0xe9SPOec43uPr8YMfnjhFF0l9Ak6PeDbVc651/mEew1c28jVTV7VICL/VJiTznnThrGgbBtfO308uen991TJ31ZW8fK6av79nEkU52X4XU7C0/3VIknk2nklNLREWFDWf+cfqmtq5QdPrmbysFyuPbbE73L6BAWBSBKZNjyPWaMGct+iLf12eupfPf8eNfua+clFUwlpLqFO0VESSTLXHlvCB7v388q6ar9L6XUrtu/l/je3cPXcUUwfked3OX2GgkAkyZw5ZQjDBqTzu9c2+l1Kr4pEHd95dBWDs9M0vXQXKQhEkkxKMMAXThjDu1v28Pam/rNozZ/f2srK8lq+e25pvx4I94KCQCQJffaokQzOSuXOVzb4XUqv2FHXxC+fW8/xR+Rz3jRNV9ZVCgKRJJSRGuTzx4/hH+/vZPm2vX6X02M/e2YdLeEoP7xA9wx0h4JAJEldNXckuekh7urjvYLFW3fz6NJyPn/8aEbnZ/ldTp+kIBBJUjnpKVw7bzTPr9nB+qp6v8vplmjU8R9PrKEoN42bThrndzl9loJAJIldd2wJmanBPtsrWLB4GyvLa7ntrElkpXk2UUK/pyAQSWIDs1K5au4onlpRwZY+NhldbWMrv3h2PbNGDeSCIztcJ0s+gYJAJMl9/rjRmBkPvfuB36V0yR0vvc/u/S384PzJGiDuIQWBSJIrzE3npAkFPLqknHAk6nc5nVJd38R9i7Zw2ewRTCke4Hc5fZ6CQES4ZNZwquub+8wKZi+uqSYcdVw3b7TfpfQLCgIR4eSJRQzMTOHhxX1jVtLn11QxanAm44uy/S6lX1AQiAipoQAXHFnMC2t2sHd/i9/ldGhfc5hFG3Zx2qQijQ30EgWBiADwmdnDaYlEeXxZhd+ldOjV9TW0RKKcPlnLT/YWBYGIADB52ABKh+Ym/Omh59dUMSgrlVmjBvpdSr+hIBCRD10yazgry2tZV1XndyntaglHeXldNadOKiQY0Gmh3qIgEJEPXTijmJSg8XCCLmX59uZd1DeFOa1Up4V6k4JARD40KCuVkycW8ujScloT8J6CF9bsICMlyPFH5PtdSr+iIBCRj7jsqBHsamjhhTU7/C7lI5xzPL96ByeMzyc9Jeh3Of2KgkBEPuJT4wspzsvggbe3+l3KR6wsr6WqrkmnhTygIBCRjwgGjMuPHsEbG3axOYEmont+9Q4CBqdMLPS7lH5HQSAiH3Pp7BGEAsZD7yTORHSvrK/mqJJBDMxK9buUfkdBICIfU5ibzmmlRSwo20ZTa8TvcnDOsWVnA6XDcv0upV9SEIhIu66cM4o9+1t5dlWV36VQ29hKQ0uE4rwMv0vplzwLAjO7x8yqzWzVYd4faGaPmtkKM3vHzKZ4VYuIdN2xYwdTMjgzIQaNy/c2AigIPOJlj+Be4MwO3v82sMw5Nw24Bvi1h7WISBcFAsYVc0by7pY9vq9pXLG3CYBhCgJPeBYEzrnXgN0dNCkFXo61XQeUmFmRV/WISNddMmsEqcEAD/rcKyjfsx+A4oEKAi/4OUawHLgYwMyOBkYBw9traGY3mFmZmZXV1NTEsUSR5DYoK5Wzpw5h4ZJy9reEfaujoraJtFCAwbpiyBN+BsHPgDwzWwZ8BVgKtHt5gnNuvnNutnNudkFBQTxrFEl6V84dRX1zmCeX+zc9dfmeRorzMrT+gEd8CwLnXJ1z7jrn3JG0jREUAJv8qkdE2jd71EDGF2Xz4Nv+3VNQvrdR4wMe8i0IzCzPzA708z4PvOacS8y5b0WSmJlx5ZxRLN9ey8rttb7UULG3kWF56b7sOxl4efnoQ8CbwAQz225m15vZjWZ2Y6zJJGCVma0HzgK+6lUtItIzF84oJj0lwIPvxH/QuDkcobq+meK8zLjvO1mEvNqwc+7yT3j/TWC8V/sXkd4zICOF86cP4/FlFdx29iRy01Pitu+q2gOXjqpH4BXdWSwinXLlnFHsb4nw+NLyuO63fE/sZjJdOuoZBYGIdMq04QOYUpzLA29/gHMubvvVXcXeUxCISKccGDReV1XPkg/2xm2/B+4qHjJAp4a8oiAQkU47f/owstNC/CWO01OX791PYU4aaSGtSuYVBYGIdFpWWojTS4t4bnUVLeH4rGlcsbdJ9xB4TEEgIl1y7vSh1DWFeX1DfKZ7Kd/bqIFijykIRKRLjhtXQG56iKeWV3q+L+dcWxCoR+ApBYGIdElqKMAZk4fwwpodnq9etnNfCy3hKMM0UOwpBYGIdNm504dR3xzmtfe8PT1UceDS0YG6q9hLCgIR6bJjxw5mYGYKT6/09vTQgSDQXcXeUhCISJelBAOcOWUIL3p8eujAzWTDNc+QpxQEItIt50wdRkNLhFfWVXu2j/K9jWSlBsnN8GxaNEFBICLdNHfMIAZnpfKUh6eHyve0XTqqBWm8pSAQkW4JxU4Pvby22rNlLCtqtSBNPCgIRKTbzp02jMbWCC+u9eb0kO4qjg8FgYh029GjBzEkN51Hl2zv9W3vbwmzu6FFN5PFgYJARLotGDAunFHMa+/vpKa+uVe3fWDWUQWB9xQEItIjF88sJhJ1PLm8ole3++E6BJpnyHMKAhHpkfFFOUwelsujvbxy2T9vJlMQeE1BICI9dtGMYlaW1/L+jvpe22bF3kaCAaMoJ63XtintUxCISI+df+QwggFjYS/2Crbs2s/QAemEgvo15TUdYRHpscKcdI4/Ip/Hl5YTjfbOesZrK+uYOCS3V7YlHVMQiEivuGhGMRW1Tby1eVePt9XUGmFTzT5Kh+b0QmXySRQEItIrTi8dQnZaiEeX9Kk/wYsAAAkCSURBVPz00Hs76ok6mDRUPYJ4UBCISK/ISA1y1pQh/G1lJY0tPZuRdE1FHaAgiBcFgYj0mgtnFLfNSLq+Z1NOrK2sIys1yMhBmn46HhQEItJr5o4ZTH52Wo9vLltbWc/EobkEApp1NB48CwIzu8fMqs1s1WHeH2BmT5rZcjNbbWbXeVWLiMRHMGCcPXUIL6+rZl9z92Ykdc6xtrKOSRoojhsvewT3Amd28P5NwBrn3HTgROB2M0v1sB4RiYPzpg+jORzlpbU7uvX57XsaqW8Oa3wgjjwLAufca8DujpoAOda24kR2rK03k5qLSNzMGjmQIbnp3T49tKaybaC4VEEQN36OEdwJTAIqgJXAV51z0fYamtkNZlZmZmU1NTXxrFFEuigQMM6ZNpRX36uhtrG1y59fW1mHGUwYolND8eJnEJwBLAOGAUcCd5pZu38COOfmO+dmO+dmFxQUxLNGEemG86YPozXieH51VZc/u6aijtGDs8hM1TrF8eJnEFwHLHRtNgCbgYk+1iMivWT68AGMGJTBkyu6vp7x2qo6jQ/EmZ9B8AFwCoCZFQETgE0+1iMivcTMOHfaMN7YsJPdDS2d/lx9UyvbdjdSOkxBEE9eXj76EPAmMMHMtpvZ9WZ2o5ndGGvyI+BYM1sJvATc6pzb6VU9IhJf504bSiTqeHZV508Pratqm8Zal47Gl2cn4Zxzl3/C+xXA6V7tX0T8VTo0lzH5WTy+rJwr5ozs1Gc0tYQ/dGexiHjCzLhk9nDe3ryb1RW1nfrM2so68jJTGJKb7nF1cjAFgYh45so5o8hOC/HbVzs3/Le2so7Sobm03V4k8aIgEBHPDMhI4cq5I3l6RQVbdzV02DYcibKuql6nhXygIBART10/bzShQID5r3XcK9iyq4HmcFRB4AMFgYh4qjA3nU/PGs6Cxduprm9qt41zjgff3gZoagk/KAhExHNfPGEM4UiUP76x5WPvhSNRblu4knve2MzlR4/QpaM+UBCIiOdK8rM4e+pQ/vzmVuqa/jn/UFNrhC89sIS/vLuNm04ay08umqqBYh9oMg8RiYsbPzWWp1ZUctnv3mLogHTSQgG27trPmso6vn9eKdfNG+13iUlLQSAicTGleAD/euJY3t28m5r6Zppa29Y1vuPyGZw/fZjP1SU3BYGIxM2tZ2peyUSkMQIRkSSnIBARSXIKAhGRJKcgEBFJcgoCEZEkpyAQEUlyCgIRkSSnIBARSXLmnPO7hi4xsxpgazc/PgDo3FJJnW/fUZuuvteZ1/KBeK/t3NXj1tPP67j3/Jh3Zxs67v37uI9yzhW028o5lzQPYH5vt++oTVff68xrQFmiHzcdd/+PuY67jntXakm2U0NPetC+ozZdfa+zr8VbT2vQce+63ti/jnvXJcNx/5g+d2oo2ZlZmXNutt91JBsdd3/ouMdHsvUI+oP5fheQpHTc/aHjHgfqEYiIJDn1CEREkpyCQEQkySkIRESSnIKgHzGzC83sbjP7PzM73e96koWZjTGzP5jZw37X0t+ZWZaZ3Rf7f36l3/X0FwqCBGFm95hZtZmtOuT1M81svZltMLNvdbQN59xjzrkvADcCl3lZb3/RS8d9k3Puem8r7b+6+D24GHg49v/8/LgX208pCBLHvcCZB79gZkHgLuAsoBS43MxKzWyqmT11yKPwoI/+e+xz8snupfeOu3TPvXTyewAMB7bFmkXiWGO/psXrE4Rz7jUzKznk5aOBDc65TQBm9hfgAufcT4FzD92GmRnwM+AZ59wSbyvuH3rjuEvPdOV7AGynLQyWoT9ke40OZGIr5p9//UDbD0FxB+2/ApwKXGJmN3pZWD/XpeNuZoPN7LfADDO7zeviksThvgcLgU+b2f/i/3QU/YZ6BP2Ic+4O4A6/60g2zrldtI3LiMeccw3AdX7X0d+oR5DYyoERB309PPaaeEvH3X/6HsSRgiCxvQscYWajzSwV+CzwhM81JQMdd//pexBHCoIEYWYPAW8CE8xsu5ld75wLA18GngPWAn91zq32s87+Rsfdf/oe+E+TzomIJDn1CEREkpyCQEQkySkIRESSnIJARCTJKQhERJKcgkBEJMkpCKTfMLN9cd7f72MzYsZzn7eYWWY89yn9n+4jkH7DzPY557J7cXuh2I1NcRObQdacc9HDvL8FmO2c2xnPuqR/U49A+jUzKzCzR8zs3dhjXuz1o83sTTNbamaLzGxC7PVrzewJM3sZeMnMTjSzv5vZw2a2zsweiP2yJvb67NjzfWb2YzNbbmZvmVlR7PWxsa9Xmtl/ttdrMbOS2AIs9wOrgBFm9r9mVmZmq83sB7F2NwPDgFfM7JXYa6fH/h1LzGyBmfVaEEoScc7poUe/eAD72nntQeC42PORwNrY81wgFHt+KvBI7Pm1tE15PCj29YlALW2TngVomwrhwPb+Tttf5wAOOC/2/BfAv8eePwVcHnt+42FqLAGiwNyDXjuw/2BsP9NiX28B8mPP84HXgKzY17cC3/P7+6BH33toGmrp704FSmN/xAPkxv5qHgDcZ2ZH0PZLPOWgz7zgnNt90NfvOOe2A5jZMtp+cb9+yH5aaPulD7AYOC32/BjgwtjzB4H/OkydW51zbx309aVmdgNtU8UPpW2VrhWHfGZu7PU3Yv++VNqCSqRLFATS3wVo+0u76eAXzexO4BXn3EWx1bH+ftDbDYdso/mg5xHa/7lpdc65T2jTkQ/3aWajgW8ARznn9pjZvUB6O58x2kLr8i7uS+QjNEYg/d3ztK3cBoCZHRl7OoB/zm9/rYf7fwv4dOz5Zzv5mVzagqE2NtZw1kHv1QM5B217npmNAzCzLDMb3/OSJdkoCKQ/yYxNY3zg8TXgZmC2ma0wszX8cyWxXwA/NbOleNszvgX4mpmtAMbRNt7QIefccmApsI6200lvHPT2fOBZM3vFOVdDW4g9FNv+m8DE3i1fkoEuHxXxUOya/0bnnDOzz9I2cHyB33WJHExjBCLemgXcGbvkdC/wLz7XI/Ix6hGIiCQ5jRGIiCQ5BYGISJJTEIiIJDkFgYhIklMQiIgkOQWBiEiS+/8VVN/o7Usi6wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "reset_seed(SEED)\n",
    "batch_size = 8\n",
    "accumulation_steps = DESIRED_BATCH_SIZE//batch_size\n",
    "\n",
    "# re-create `DataLoader` to apply the new `batch_size`\n",
    "trainloader = DataLoader(trainset, batch_size=batch_size, shuffle=True)\n",
    "\n",
    "device = torch.device('cuda')\n",
    "model = ConvNet()\n",
    "model = model.to(device)\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.5)\n",
    "criterion = nn.NLLLoss()\n",
    "\n",
    "lr_finder = LRFinder(model, optimizer, criterion, device='cuda')\n",
    "lr_finder.range_test(trainloader, end_lr=10, num_iter=100, step_mode='exp', accumulation_steps=accumulation_steps)\n",
    "lr_finder.plot()\n",
    "lr_finder.reset()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Validation loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e6e8e96fce5d480a93fade2b717f5787",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Learning rate search finished. See the graph with {finder_name}.plot()\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEOCAYAAACXX1DeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deXxc5X3v8c9vNBrtkq3FC5J3Y2yzxSDA4BAcoBSyQAikgSwUQuo6zUaT9KZJc9vb2yVNk5vepqRJSSBOcoEmYQlJGgpZIMSYzRiz2GbxLq+S5U2ytll+948ZLxh5kGwdHc3R9/16zUtn5jxz5ufzyPPT85zneY65OyIiIscSCzsAEREZ2ZQoREQkLyUKERHJS4lCRETyUqIQEZG8lChERCSvwBKFmU0ys0fMbLWZrTKzT+cpe46Zpczs2qDiERGR4xMP8Ngp4LPuvsLMqoBnzexX7r76yEJmVgR8BXg4wFhEROQ4BdaicPft7r4it90BrAEa+yn6SeBeoDWoWERE5PgNyzUKM5sKzAOeOur1RuBq4FvDEYeIiAxekF1PAJhZJdkWwy3uvv+o3f8X+Ly7Z8ws3zEWAYsAKioqzp49e3ZQ4YqIRNKzzz67y90bjue9FuRaT2ZWDPwCeMjdv97P/g3AwQxRD3QBi9z9p8c6ZnNzsy9fvjyIcEVEIsvMnnX35uN5b2AtCss2EW4H1vSXJADcfdoR5ZcAv8iXJEREZPgF2fW0APgw8KKZrcy99kVgMoC7fzvAzxYRkSESWKJw96Uc7lYaSPkbg4pFRESOn2Zmi4hIXkoUIiKSlxKFiIjkpUQhIsOiraOXJ9a1s21vN5mMbsFcSAKfcCciAvDXD7zEgy/tACARjzGltpwpdeVMqi1nSm05k+vKmVJXQdPYMkriRSFHK0dSohCRYbGurZOzJo/hmrOb2NTexYZdB2jZ3cWyde109aUPlTODk2rKmFZfwbT6CqY3ZH+ePL6Kk2pKybeKgwRDiUJEAufutOzu5vpzJ/PB86a8YV/7gT42tR9gU3sXm9q72Nh+gI3tXfx05VY6elKHylaWxJk5rpJTxlcxa0IVsydUMWt8FQ1VJcP9TxpVlChEJHDtB/roTqaZVFv2hn1mRn1lCfWVJZw9pfZ1+w4mkXWtnbzW2slrOzt4dWcnv16zkx8tbzlUrrYiwcxxlcwaX8ms8VXMbKhkxrhKxlWVDLgF0tbRS08yzUljyiiKqdVyJCUKEQlcy+4uACaNLR/U+45MIudNr3vdvl2dvby6o4OXd3TwWms2gTywctvrWiBVpXGmN1QyPdeNNbW+gml1FUypL6e6tJhkOsNv1rTy4+UtPPpKKxmHRFGMKXXlTK2vYGruusmUunKm1FYwoaaURHz0jQFSohCRwLXs6QZgUu3gEkU+9ZUl1M8s4YKZ9Ydec3d27u9lXVsn69o6Wdvayfq2Azy9YTf3P7f1de+vq0iQcWdPV5Lx1SV8bOEMmsaWs3HXATbkHo+92kZvKnPoPWYwobqUxjFlTKgppaGqhIaqEsZVleZ+ljC+upSx5cWRupaiRCEigTvYomga+8aup6FkZkyoKWVCTSkLjkggAN19aTbtPsDGXV1syl0D6UmmedcZE7loVgPxoje2FDIZZ2dHD5vau9i8u4ute7rZsqebLXu6WLVtP20dvXT2pt7wvuIiY1xVKeOqs8mjvrKEuooEtRUJxlYkKE/EKSsuoixRRHmiiIpE/NB2WXERsRHW9aVEISKB27Kni7qKBBUl4X3llCWKmD2hmtkTqgf8nljMmFhTxsSaMuYf1fV1UFdfiraOXlo7emnd30trRw+tHb3s3N9D6/5eNuw6wPKNe9jd1cdA7+pQEo9RniiisjROXUUJ9ZUJ6ipKuGTOOC47dcKA4x8qShQiEriW3d00DWG300hSnogzpS7OlLqKvOXSGWdvVx97upJ096Xp6kvRlUznttN096U40Jd93pNM051Ms787SfuBPrbt7eHFrftoHFumRCEi0dSyp4vTG2vCDiNURTGjrrKEusrCG8o7+i7fi8iwSmecrXu6h/RCtgwvJQoRCdT2fd2kMj7oobEycihRiEigWnYfHBob7IgnCU5gicLMJpnZI2a22sxWmdmn+ylzlZm9YGYrzWy5mb01qHhEJBwte7JDYyer66lgBXkxOwV81t1XmFkV8KyZ/crdVx9R5jfAz9zdzewM4MfA7CCCWdvawUOrdmIGMTNih37mtmOGmVFkRlHs8L6imBGLHX69KBY7tD8eixGLQTwWoyiWLRs/4me8KHb4eZFRHItRlPsZL8qWidKkHJH+bNndRczgpDFqURSqIO+ZvR3YntvuMLM1QCOw+ogynUe8pQIIbJH6V3Z08tWHXgnq8MetuMgoLoodeiSKjOL4Ec/jMUpyPxPxGCVH/CyJF1FanP1ZEo9RliiipLiI0tx2WXH2UZqbyFNeHKc0EctO7hmBk3okmlr2dDOxpozifia0SWEYluGxZjYVmAc81c++q4EvA+OAdwYVwxWnTeDVv7+CjDvukHEn7Y5nDm9n3MlkyG5nss/Th35yaDuVOfx6Kp3dTruTzmQOPT9YJpnOZH9mnHQ6QyrjJNNOKp0hmc6QzBzcdnpTmUOv96Uz9KU89zM75npvd4beZHZfTzJNXypDbyq7fTz3gSkrLqKipIjyRJyKkjiVJUW5n3GqSrM/K0uKqSrNPq8uK6a6tJjqsjjVpcWMKS+msiSuVpHk1bK7K/AZ2RKswBOFmVUC9wK3uPv+o/e7+/3A/Wb2NuDvgEv7OcYiYBHA5MmTjyuOWMxIRPQvaPds8ulJZSfq9PRl6Ekdnshz5HZ3X4quvnRuYk+Kzt5sEjrQm6KzN0V7Zx+b27vo6E3R2ZOiO5nO+9nxmDGmvJgx5QlqyxOMrSimtiK7XEFdZSK7dEFlgnFVJTRUlVJdqsQy2rTs6eLCkxvCDkNOQKCJwsyKySaJO939vnxl3f0xM5tuZvXuvuuofbcBtwE0NzfrHopHMTMScSMRj1FdWjykx06lM3T2pujoSbGvO8n+niT7u1Ps706yrzvJntxM071dfew+0MeGXQd4dtMedh/o67eVUxKP0VBVwoTqUsbXlDKxupSJY8pozD2axpYxJmILqo1mPck0O/f3amhsgQssUVj2f/rtwBp3//oxyswE1uUuZp8FlADtQcUkgxcvijGmPMGY8gSTBvG+TMbZ252kvbOXts5e2joOP3bu72HH/h5Wbd3Hb9bspCeZed17q0ri2dtj5pZ4nt5QwYyGCqbXVzK2IjG0/0AJ1Na9GhobBUG2KBYAHwZeNLOVude+CEwGcPdvA9cAN5hZEugG3u8+0GWzZCSLxYza3GqZJ4+vOmY5zy3zvG3v4VU5W3ZnV+p8ZWcHv16zk2T68K9EfWUJs3N3Nps9sZozmmqY0VCpG82MUIfuQ6GhsQUtyFFPS4G8/3vd/SvAV4KKQUY+s8MJ5bR+1gJKpTNs2dPN+l2drGs9wCs7O3hlRwc/fHLTofsElCeKOO2kGuZNGcP8aXU0Tx1L1RB3wcnxOXQfCnU9FTQtCigjWrwolr3TWH0FFx8xwyadcTbs6uSFLft4Ycs+nt+ylzuWbuA/freemMFpjTVcNKuBS+aM54zGGg0FDsmW3V0k4jHG6Z7WBU2JQgpSUcyYOa6KmeOqeO9ZTUD2xjTPbd7Dkxt288S6XXzzkbX822/X0lBVwqVzxnP1vEbOmTpWF8qHUcueLprGlClRFzglComMskQRF8ysz94a8w9msedAH4++2sqv17TywMqt3P30ZibVlvHeeU28r7mJJnWHBC7K96EYTZQoJLLGViS4el4TV89roqsvxUOrdnDfiq1847evcesja3nH6RNZdOF0Tm8a3fdJCFLLni7O0PkteEoUMiqUJ+KHksbWvd18f9lG7npqMz9/fhvzp9fyl1fM4S2TxoQdZqR09CTZ25XUiKcI0OIrMuo0jinji++Yw7IvXMxfvWMO69oO8J5vPs7/uOd52jp6ww4vMg4tL64uvoKnRCGjVnVpMX/ytuk88rmF/OlF07n/ua1c/LVH+cETG9F0nhO3fV82UTRqnaeCp0Qho15lSZwvXDGHh255G/OmjOWvH1jFJ+9+jq6+VNihFbQj57lIYVOiEMmZ3lDJ9286h89fPptfvridq7+5jI27DoQdVsFKprOJIq6hsQVPiULkCGbGxxbO4PsfOZedHT28+9alLFu7683fKG/Ql2tR6D4UhU81KNKPC09u4OefeCsTa0q5+fvLeXrD7rBDKjgH1+hKxPU1U+hUgyLHMKm2nDs/Op+JY0r5yJJnWNmyN+yQCsrBrie1KAqfalAkj4aqEu766HxqKxLccPtTrNq2L+yQCsbhRKFrFIVOiULkTUyoKeXOj55HRUmcG25/mh37esIOqSAc7HpSi6LwqQZFBmBSbTk/vPk8uvrS/MU9z5M5npuUjzLqeooO1aDIAM0cV8lfvXMOv39tFz98clPY4Yx4yXSGmKGbSkWAEoXIIHzwvMksPKWBf/zlGta2doYdzoiWTLtaExERWC2a2SQze8TMVpvZKjP7dD9lPmhmL5jZi2a2zMzODCoekaFgZvzzNWdQnijiz3+08lD3irxRMp0hoUQRCUHWYgr4rLvPBeYDHzezuUeV2QBc5O6nA38H3BZgPCJDYlx1Kf949em8uHUft/52bdjhjFjJdIa4RjxFQmCJwt23u/uK3HYHsAZoPKrMMnffk3v6JNAUVDwiQ+mK0yfy7jNP4lu/W8e2vd1hhzMiJdMZdT1FxLDUoplNBeYBT+UpdjPw4DHev8jMlpvZ8ra2tqEPUOQ4fP7yU8Dh6796NexQRqS+lK5RREXgtWhmlcC9wC3uvv8YZd5ONlF8vr/97n6buze7e3NDQ0NwwYoMQtPYcm5cMJV7V2xhzfZ+f7VHtVQmo+U7IiLQWjSzYrJJ4k53v+8YZc4Avgtc5e7tQcYjMtT+bOEMqkri/NODL4cdyoiT7XrSNYooCHLUkwG3A2vc/evHKDMZuA/4sLur/S4FZ0x5gk9cPJPfvdrG41pl9nX6Uk48phZFFARZiwuADwMXm9nK3OMdZrbYzBbnyvw1UAf8e27/8gDjEQnEDedPpXFMGV9+cI1mbB8hmc5QrK6nSIgHdWB3XwrkbXe6+0eBjwYVg8hwKC0u4nN/OIs//9HzPPjSDt55xsSwQxoRsvMo1PUUBUr3IkPgqjMbmVJXzu1L14cdyoiR0szsyFAtigyBWMy46YKprNi8l+c273nzN4wCfekMcSWKSFAtigyRa5snUVUS547HN4YdyoigrqfoUKIQGSKVJXHef84kfvnidrbv02xtzcyODtWiyBD64wum4u784AktQ67VY6NDtSgyhCbVlvOHp07grqc209WXCjucUKlFER2qRZEh9pG3TmNfd5L7VmwNO5RQaWZ2dChRiAyx5iljOb2xhjse34D76J2Ap66n6FAtigwxM+PGC6ayvu0Ay9aN3uXLkil1PUWFalEkAO88YyJjyov5f6P43tp96QzFcXU9RYEShUgASouL+KPmSTy8eic79/eEHU4oUhnXrVAjQrUoEpAPnDuZdMa5++nNYYcy7NIZJ53R6rFRoVoUCcjU+greNquB/3y6hVQ6E3Y4wyqZ+/eq6ykalChEAvSh8yazY38Pv17TGnYow+pgolDXUzSoFkUCdPHscUysKR11F7VT6eywYI16igbVokiA4kUxrj93MkvX7mLDrgNhhzNsDrYo4ppwFwlKFCIBu+6cScRjNqpaFX0Hr1GoRREJQd4ze5KZPWJmq81slZl9up8ys83sCTPrNbPPBRWLSJjGVZdyxekT+fEzLXT2jo71n5K5riddo4iGIGsxBXzW3ecC84GPm9nco8rsBj4FfC3AOERC95EFU+noTXHP8pawQxkWSbUoIiWwWnT37e6+IrfdAawBGo8q0+ruzwDJoOIQGQnmTR7LvMlj+N6yjWQy0V//6XCi0DWKKBiWdG9mU4F5wFPH+f5FZrbczJa3tbUNZWgiw+YjC6axqb2L374c/aGySY16ipTAa9HMKoF7gVvcff/xHMPdb3P3ZndvbmhoGNoARYbJ5adNYGJNKd9btiHsUAKnrqdoCbQWzayYbJK4093vC/KzREa64qIYN5w/lcfXtvPyjuP6m6lgJFPqeoqSIEc9GXA7sMbdvx7U54gUkuvPnURpcYzvLd0YdiiBOjQ8Nq4WRRQEWYsLgA8DF5vZytzjHWa22MwWA5jZBDPbAnwG+JKZbTGz6gBjEgnVmPIE15zVxP0rt9Le2Rt2OIFJaXhspMSDOrC7LwXytjvdfQfQFFQMIiPRB8+bwp1PbeahVTv5wHmTww4nEJqZHS1K9yLDbM7EKibXlvPw6h1hhxIYzcyOFtWiyDAzMy6bO55la9vp6InmFCLNzI4W1aJICC47dQJ96QyPvhLNeUEaHhstqkWREJw9ZSx1FQkeXr0z7FACkdI1ikhRohAJQVHMuHTOeB55uZXeVDrscIZcn2ZmR4pqUSQkl506ns7eFE+u3x12KENOd7iLFtWiSEgWzKynPFHEQ6uiN/pJM7OjRYlCJCSlxUUsPKWBX63eGbkVZZPpDGbZLjYpfEoUIiG6bO4E2jp6Wbllb9ihDKlkximOxciu5COFTolCJERvnz2OeMx4eFW0Rj8lUxl1O0WIEoVIiGrKijl/Rh2/itgs7WQ6owUBI0Q1KRKyi2Y1sK7tANv3dYcdypDpS7uGxkaIalIkZBfMqAfg8bXtIUcydFLpjIbGRohqUiRksydUUVeRYNnaXWGHMmSS6YxmZUeIEoVIyGIx44KZ9Sxduwv3aAyTTarrKVJUkyIjwIIZdbR29LK2tTPsUIZEXzqjRBEhQd4KdZKZPWJmq81slZl9up8yZmbfMLO1ZvaCmZ0VVDwiI9mCmdnrFEsj0v2UTGdIqOspMgaUKMxshpmV5LYXmtmnzGzMm7wtBXzW3ecC84GPm9nco8pcAZyceywCvjWo6EUiYlJtOVPqyiNzQTulrqdIGWhN3gukzWwmcBswCbgr3xvcfbu7r8htdwBrgMajil0F/MCzngTGmNnEwfwDRKLighn1PLm+/dAS3YWsTxezI2WgiSLj7ingauDf3P0vgAF/oZvZVGAe8NRRuxqBliOeb+GNyURkVHjrzHo6e1M8v2Vf2KGcsKSuUUTKQGsyaWbXA38M/CL3WvFA3mhmlWRbJLe4+/7BhwhmtsjMlpvZ8ra2aN4RTOT8GXWYweMRuE6R1DyKSBloTd4EnA/8g7tvMLNpwA/f7E1mVkw2Sdzp7vf1U2Qr2W6sg5pyr72Ou9/m7s3u3tzQ0DDAkEUKS21FglNPqo5GokjpGkWUDKgm3X21u3/K3e82s7FAlbt/Jd97LLts5O3AGnf/+jGK/Qy4ITf6aT6wz923D+YfIBIlC2bUs2LzHrr6UmGHckKSGV2jiJKBjnp61MyqzawWWAF8x8yO9eV/0ALgw8DFZrYy93iHmS02s8W5Mr8E1gNrge8Af3Z8/wyRaFgws55k2nl6Q2Hf9U5dT9ESH2C5Gnffb2YfJTtK6W/M7IV8b3D3pUDePyk8Ow314wOMQSTyzplaS2lxjIdX72ThKePCDue4qespWgZak/HcsNU/4vDFbBEZYmWJIq44bSI/X7mN7r502OEct+wy4+p6ioqBJor/DTwErHP3Z8xsOvBacGGJjF7va26iozdV0PfS1hIe0TLQi9k/cfcz3P1juefr3f2aYEMTGZ3mT6ujaWwZP3m25c0Lj1CamR0tA72Y3WRm95tZa+5xr5k1BR2cyGgUixnXnt3EsnXtbNnTFXY4xyU74U5dT1Ex0JT/PbJDWU/KPX6ee01EAnDNWU24w73PvmFa0YiXyTipjFoUUTLQmmxw9++5eyr3WAJo5ptIQCbVlnPBjDruWdFCJlNY96hIZrJrVSlRRMdAa7LdzD5kZkW5x4eAaCxzKTJCva+5iZbd3TxVYHMqkulsYtM8iugYaE1+hOzQ2B3AduBa4MaAYhIR4PJTJ1JVEi+4i9oHV7/VzOzoGOiop03ufqW7N7j7OHd/D6BRTyIBKksU8a4zJ/LLF7fT0ZMMO5wB60ur6ylqTqQmPzNkUYhIv649exI9yQwPvlg4cyrU9RQ9J1KTaleKBOysyWOYVl/BPSu2hB3KgCVTuRaFZmZHxokkisIaiiFSgMyMa85q5OkNu2nZXRhzKlK5UU/xmFoUUZG3Js2sw8z29/PoIDufQkQCdvVZTZjBvQXSquhLZf+G1DWK6Mhbk+5e5e7V/Tyq3H2gK8+KyAloHFPG+dPruG/FVrILLo9sydzF7IS6niJDKV+kAFxzVhObd3fxzMY9YYfyppIa9RQ5qkmRAnD5aRMoTxRx77Mjv/tJw2OjRzUpUgAqSuJccdpE/uvF7SP+PhWp9MFrFOp6iorAEoWZ3ZFbafalY+wfm1uR9gUze9rMTgsqFpEouObsRjp7Uzy8emTPqVDXU/QEWZNLgMvz7P8isNLdzwBuAP41wFhECt78aXU0jinjgZXbwg4lLyWK6AmsJt39MSDfamZzgd/myr4MTDWz8UHFI1LoYjHj0jnjeGJdO72pkdv91JfW8NioCbMmnwfeC2Bm5wJTAN0MSSSPC09uoDuZ5tlNI3f008GZ2VrCIzrCrMl/AsaY2Urgk8BzQL9/JpnZIjNbbmbL29rahjNGkRFl/ow64jFj6Wu7wg7lmA7NzNbF7MgILVG4+353v8nd30L2GkUDsP4YZW9z92Z3b25o0P2SZPSqLIlz1uSx/H4EJwp1PUVPaDVpZmPMLJF7+lHgMXffH1Y8IoXiwpPreWnbPto7e8MOpV/qeoqeIIfH3g08AZxiZlvM7GYzW2xmi3NF5gAvmdkrwBXAp4OKRSRKLpzVgDs8vm5k3mTy0KgnLeERGYGt1+Tu17/J/ieAWUF9vkhUnd5YQ01ZMUtfa+PKM0fe2pwHE4VWj40O1aRIgSmKGQtm1vH713aNyEUCk5qZHTlKFCIF6MKTG9i+r4d1bZ1hh/IGyXSG4iLDTIkiKpQoRArQW2fWA/DYqyNv9FM2UeirJUpUmyIFaFJtOdPqK/j9ayNvXlEy7UoUEaPaFClQF55cz5Prd4+45Tz6cl1PEh1KFCIF6uByHis27Q07lNdJqespclSbIgXqvOm1xAyeWD+y5lOo6yl6VJsiBaq6tJhTT6rhyRGWKNT1FD1KFCIFbP70Wla27KUnOXKuUyRT6nqKGtWmSAE7b1odfakMK1tGznWKVEZdT1Gj2hQpYOdMq8WMEdX9lFTXU+QoUYgUsJqyYuZOrOap9fluJjm8+tT1FDmqTZECd960OlZs3jNi5lMk0xkScX21RIlqU6TAzZ9eS28qw/Mt+8IOBdDw2ChSbYoUuHNH2HWKZDpDPKZrFFGiRCFS4MaUJzhlfBVPbRg5iaJYXU+RotoUiYD50+t4dtMe+nK3IQ1TMu26DWrEBHkr1DvMrNXMXjrG/hoz+7mZPW9mq8zspqBiEYm6+dNr6UlmeGFL+PMpNDw2eoJM+0uAy/Ps/ziw2t3PBBYC/8fMEgHGIxJZ506rA0bGdYpkOkNcLYpICaw23f0xIN/gbgeqLHsbrMpc2VRQ8YhEWW3FwesU4c+nUNdT9IRZm7cCc4BtwIvAp9293w5WM1tkZsvNbHlb28i7UYvISHD+jDqe2bibPQf6Qo1DXU/RE2ai+ENgJXAS8BbgVjOr7q+gu9/m7s3u3tzQ0DCcMYoUjOvOnURPMsMPntgUahy6FWr0hFmbNwH3edZaYAMwO8R4RAra7AnVXDJ7HEuWbaCrL5xeXHfXhLsICrM2NwOXAJjZeOAUYH2I8YgUvD97+wz2dCX5z6dbQvn8ZNoB1PUUMUEOj70beAI4xcy2mNnNZrbYzBbnivwdcIGZvQj8Bvi8u+8KKh6R0eDsKbWcO7WW7/5+fShzKlKZ7GeqRREt8aAO7O7Xv8n+bcBlQX2+yGj1sbfP4KbvPcPPnt/GtWc3DetnJ1MHWxRKFFGi2hSJmIWzGpg9oYpv/24dmYwP62f3pXMtCi3hESmqTZGIMTM+tnAGa1s7eXj1zmH97GQuUSR0jSJSlChEIuidp09kekMF/+tnq9g9jPMqDiaKeExfLVGi2hSJoHhRjG9cN4/dB/r4zI9XDlsX1KFRT+p6ihTVpkhEndZYw/9891wefaWN/3hseEaeq+spmpQoRCLsQ+dN5l1nTORrD7/CMxuDXwfqYKLQqKdoUW2KRJiZ8eX3ns6ksWV88q7n2NeVDPTzDl2jUKKIFNWmSMRVlRZz6wfOYmdHD//+6NpAP0szs6NJiUJkFDitsYar5zWyZNlGtu/rDuxzDl+j0FdLlKg2RUaJP790Fu7wr79+LbDP0DWKaFJtiowSk2rL+eD8yfx4eQtrWzsD+Yw+LeERSapNkVHkE2+fSXkiztceeiWQ4x9uUegaRZQoUYiMInWVJfzJhdP571U7eG7zniE/vlaPjSbVpsgoc/OF06irSPC1h4e+VXFo9VjNzI4U1abIKFNZEmfR26bz+Np2Vm3bN6TH7lPXUyQpUYiMQtedM5my4iKWPL5xSI+r4bHRpNoUGYVqyou55uxGHli5jV2dvUN2XM3MjqYgb4V6h5m1mtlLx9j/F2a2Mvd4yczSZlYbVDwi8no3XjCNvnSGu57aPGTH1MzsaAoy7S8BLj/WTnf/qru/xd3fAnwB+J27B79qmYgAMHNcJRfNauCHT24asvtrHxoeq/tRREpgtenujwED/eK/Hrg7qFhEpH83LZhKW0cv//XitiE5XjKdIR4zYjG1KKIk9LRvZuVkWx735imzyMyWm9nytra24QtOJOLednIDMxoq+N7jG3E/8ZsbJdNOXN1OkRN6ogDeDTyer9vJ3W9z92Z3b25oaBjG0ESiLRYzblwwjRe27OPZTSc+Aa8vldFkuwgaCTV6Hep2EgnNNWc1UlkS50fPtJzwsVKZjIbGRlCoNWpmNcBFwANhxiEympUn4lxx2gQefGkHPcn0CR0rmXK1KCIoyOGxdwNPAKeY2RYzu9nMFpvZ4pvQI9QAAAmISURBVCOKXQ087O4HgopDRN7c1fMa6exN8avVO0/oOMl0huK4rlFETTyoA7v79QMos4TsMFoRCdF50+uYUF3KT5/byrvPPOm4j9OXzmhobASpRkWEophx1byT+N2rbbSfwEztZFoXs6NINSoiQLb7KZVx/uvF7cd9jFTa1fUUQUoUIgLA7AnVzJ5Qxf3PbT3uY/SpRRFJqlEROeTqeY08t3kvG3cd3/gSdT1Fk2pURA658i0nYQY/XXl8rYpk2rUgYAQpUYjIIRNryjh/eh33P7f1uJb0SKlFEUmqURF5navnNbKpvYunNwx+Mee+tCbcRZFqVERe511nnERVaZy7nh78fSqSaS3hEUWqURF5nbJEEdec1cSDL+4Y9JyKZDqj1WMjSIlCRN7gA+dNpi+d4Z5ntwzqfUmtHhtJqlEReYNZ46s4Z+pY7n56M5nMwC9qJzO6RhFFqlER6dcHz5vCxvYulq1rH1D5vlSGrt4UJXF9rUSNalRE+nX5aRMYW17MXU9vGlD5+5/bwoG+NBedopuLRY0ShYj0q7S4iGvPbuLhVTtp3d+Tt2wyneHWR9ZyRlMNC2cpUUSNEoWIHNP1504mlfE3vfvdAyu30bK7m09dfDJmGvUUNUoUInJM0xsqWXhKA9/5/XraOvofKpvOON98ZC1zJ1ZzyZxxwxyhDIcg73B3h5m1mtlLecosNLOVZrbKzH4XVCwicvy+9M65dCfTfPnBNf3u/8UL29iw6wCfumSmWhMRFWSLYglw+bF2mtkY4N+BK939VOB9AcYiIsdp5rhK/vRtM7hvxVaeXP/6EVDpjPNvv13LKeOruGzuhJAilKAFlijc/TEg32IxHwDuc/fNufKtQcUiIifm42+fSdPYMv7nT1+iL5UBIJNx7li6gbWtnXzykpnEYmpNRFWY1yhmAWPN7FEze9bMbggxFhHJoyxRxN9eeSqvtXZy+9INLFu3iyu/uZR/+OUa5k+v5YrTJoYdogQoHvJnnw1cApQBT5jZk+7+6tEFzWwRsAhg8uTJwxqkiGRdMmc8l80dz1cfepmMQ+OYMv7l/Wdy1ZmNak1EXJiJYgvQ7u4HgANm9hhwJvCGROHutwG3ATQ3Nw9+kXwRGRJ/c+Wp7Onq45I547nxgqmUFheFHZIMgzATxQPArWYWBxLAecC/hBiPiLyJxjFl/GTxBWGHIcMssERhZncDC4F6M9sC/A1QDODu33b3NWb238ALQAb4rrsfcyitiIiEI7BE4e7XD6DMV4GvBhWDiIicOM3MFhGRvJQoREQkLyUKERHJS4lCRETyUqIQEZG8lChERCQvcy+sic5m1gYM7N6Mb1QD7BvCsvnKHGvfQF/vr1w9sOtNYhoqgzlXJ/r+4TzXA3ktqud5IOWPd7/O9eDLDvfvdIW7H9/tB9191DyA24aybL4yx9o30Nf7KwcsH4nnqpDO9UBei+p5Hkj5492vcz34soX0Oz3aup5+PsRl85U51r6Bvj6YWINwop8/Us/1QF8bLsN5ngdS/nj361wPvmzB/E4XXNfTaGZmy929Oew4ok7nefjoXA+PEz3Po61FUehuCzuAUULnefjoXA+PEzrPalGIiEhealGIiEheShQiIpKXEoWIiOSlRBEBZvYeM/uOmf3IzC4LO54oM7PpZna7md0TdixRY2YVZvb93O/yB8OOJ8oG+3usRBEyM7vDzFrN7KWjXr/czF4xs7Vm9pf5juHuP3X3PwEWA+8PMt5CNkTner273xxspNExyHP+XuCe3O/ylcMebIEbzLke7O+xEkX4lgCXH/mCmRUB3wSuAOYC15vZXDM73cx+cdRj3BFv/VLufdK/JQzduZaBWcIAzznQBLTkiqWHMcaoWMLAz/WgBHYrVBkYd3/MzKYe9fK5wFp3Xw9gZv8JXOXuXwbedfQxzMyAfwIedPcVwUZcuIbiXMvgDOacA1vIJouV6I/YQRvkuV49mGOrMkamRg7/ZQXZ/0CNecp/ErgUuNbMFgcZWAQN6lybWZ2ZfRuYZ2ZfCDq4iDrWOb8PuMbMvkX4S9hERb/nerC/x2pRRIC7fwP4RthxjAbu3k72WpAMMXc/ANwUdhyjwWB/j9WiGJm2ApOOeN6Ue02Gns718NM5Hz5Dcq6VKEamZ4CTzWyamSWA64CfhRxTVOlcDz+d8+EzJOdaiSJkZnY38ARwipltMbOb3T0FfAJ4CFgD/NjdV4UZZxToXA8/nfPhE+S51qKAIiKSl1oUIiKSlxKFiIjkpUQhIiJ5KVGIiEheShQiIpKXEoWIiOSlRCGRYWadw/x53z2elThP8DNvMbPy4fxMEc2jkMgws053rxzC48VzE5aGTW4lYHP3zDH2bwSa3X3XcMYlo5taFBJpZtZgZvea2TO5x4Lc6+ea2RNm9pyZLTOzU3Kv32hmPzOz3wK/MbOFZvaomd1jZi+b2Z25L3NyrzfntjvN7B/M7Hkze9LMxuden5F7/qKZ/X1/rR4zm5q7scwPgJeASWb2LTNbbmarzOxvc+U+BZwEPGJmj+Reuyz371hhZj8xsyFLlCKHuLseekTiAXT289pdwFtz25OBNbntaiCe274UuDe3fSPZpZhrc88XAvvILqYWI7tEwsHjPUr2r3sAB96d2/5n4Eu57V8A1+e2Fx8jxqlABph/xGsHP78o9zln5J5vBOpz2/XAY0BF7vnngb8Oux70iN5Dy4xL1F0KzM01AgCqc3911wDfN7OTyX7JFx/xnl+5++4jnj/t7lsAzGwl2S/2pUd9Th/ZpADwLPAHue3zgffktu8CvnaMODe5+5NHPP8jM1tE9lYAE8neneyFo94zP/f647l/X4JsIhMZUkoUEnUxsn+p9xz5opndCjzi7lfn7gr26BG7Dxx1jN4jttP0//8m6e7+JmXyOfSZZjYN+BxwjrvvMbMlQGk/7zGySe36QX6WyKDoGoVE3cNk7wAIgJm9JbdZw+F1+W8M8POfBK7JbV83wPdUk00c+3LXOq44Yl8HUHXEsReY2UwAM6sws1knHrLI6ylRSJSU55ZXPvj4DPApoNnMXjCz1Ry+q9c/A182s+cItmV9C/AZM3sBmEn2ekde7v488BzwMtnuqseP2H0b8N9m9oi7t5FNcnfnjv8EMHtowxfR8FiRQOXmPHS7u5vZdWQvbF8Vdlwig6FrFCLBOhu4NTekdi/wkZDjERk0tShERCQvXaMQEZG8lChERCQvJQoREclLiUJERPJSohARkbyUKEREJK//D1xMA3kwKOKgAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "reset_seed(SEED)\n",
    "\n",
    "# re-create `DataLoader` to apply the new `batch_size`\n",
    "trainloader = DataLoader(trainset, batch_size=DESIRED_BATCH_SIZE, shuffle=True)\n",
    "testloader = DataLoader(testset, batch_size=DESIRED_BATCH_SIZE*2, shuffle=False)\n",
    "\n",
    "device = torch.device('cuda')\n",
    "model = ConvNet()\n",
    "model = model.to(device)\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.5)\n",
    "criterion = nn.NLLLoss()\n",
    "\n",
    "lr_finder = LRFinder(model, optimizer, criterion, device='cuda')\n",
    "lr_finder.range_test(trainloader, val_loader=testloader, end_lr=10, num_iter=100, step_mode='exp')\n",
    "lr_finder.plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Validation loss and gradient accumulation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "77859a0bb9414a37879b1577164f69e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping early, the loss has diverged\n",
      "Learning rate search finished. See the graph with {finder_name}.plot()\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEKCAYAAAAfGVI8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd3wc1bn/8c+zu+qy5CJZsuQiF9xxFaaYBNNNbyEYEkgIv0tMSICbcnPTby7kpsMNIQmXQGIIgYTE9A7BYLAxIPfejS25SS6ybEuWtHt+f+wCxki22uystN/36zUvzc6cmXk8kvXonDlzjjnnEBGR5BXwOwAREfGXEoGISJJTIhARSXJKBCIiSU6JQEQkySkRiIgkuZDfAbRWXl6eKykp8TsMEZFOZf78+VXOufym9nW6RFBSUkJZWZnfYYiIdCpm9n5z+9Q0JCKS5JQIRESSnBKBiEiSUyIQEUlySgQiIknOs0RgZv3MbJaZrTCz5WZ2axNlLjGzJWa2yMzKzOxUr+IREZGmedl9tBH4hnNugZl1A+ab2SvOuRWHlfkX8LRzzpnZGOAxYLgXwWyvrmNJ+V4CZgQCYGbRdYOAGRb7Ggx8tO2jz9GvwdhxwQ8/H7aYEQwaoVj5UGy7mXnxzxER6TCeJQLn3DZgW2y9xsxWAsXAisPK7D/skCzAs8kRyt7fzVcfWejV6Zv1QUIIBYxQMEBK8IPP0fVQMEAoYKQEP/qcGltPCQZICX30OTUUICW2PzUUIC0U/ZoaDJCWEvzwc3ooSFpKgLRQkPSUAOkpwQ+3pacEyUgJkhJUkhKRqLi8UGZmJcB44J0m9l0G/BToDVzgVQyfGpLPs187Fecg4tyHS/RzbFvEEXEQ/nDdEf7w60fHhSMfbW+MRMs2Rj7a3vixrxEaw9H1xnAk9tXR8OH2CA3h6L6GsKMhHOFgfSONEUd9Y4SGcIT6cISGxui++sYIh2Jf2yMYMDJSgqSnBMlMjSaHjNToemZqiMzUIFlp0fWstBBZqUGy00Nkpx22pIfISU8hJz2F7PQQwYASi0hn5HkiMLNsYCZwm3Nu35H7nXNPAE+Y2aeB24GzmjjHjcCNAP37929THLmZKeRm5rbp2ETknKMh7Kj/IDk0hqlvjFDX8NHnQ40R6hrC1DVEv9Y2hGOfo+sH66P7ausbP/xcWx9mz8Faausb2X8oTG19Iwfqwy2KKTstRG5GCjkZKeRmhOiekUr3zBS6Z6bSIzOFHlmp9MpKpWdWKnnZaeRlp5GRGvT4TonIsZiXU1WaWQrwLPCSc+7OFpTfAExyzlU1V6a0tNRpiIn4ikQctQ1h9h9qjC510a81dQ3sq2ukpi62XttIdW0D1bUN7KttYG9tPXsONrD3YD0N4aZ/zrJSg+R1S6N3tzR656RT0C2dwtw0CnLSKe6eQZ/uGRR0SyMUVAc3kfYws/nOudKm9nlWI7BoA/QDwMrmkoCZDQHWxx4WTwDSgF1exSRtEwhYtHkoLURBG453znGgPsyeA/XsOlDP7gOHqNpfT9X+Q1TVRL/u2FfHyq37mLVvJwePqIEEDPrkZtC3Rwb9embSr0cmJXmZlPTKoiQvi9yMlI75h4okKS+bhiYD1wJLzWxRbNt3gf4Azrl7gSuA68ysAagFrnJeVlHEF2b24XOFfj0zj1rWOUfNoUZ2VNdRsbeWbdV1bN1bS/meWrbsPsibayvZse/Qx47plZXK0IJuDCvsxtCCbgzv042RfXJIT1Gzk0hLeNo05AU1DUldQ5jNuw+yseoAm6oOsKHyAGt21rBme82HzzNCAWNEnxzG9stl4oAenDI4j4KcdJ8jF/GPL01DIl5JTwkytCD61//hIhHH1upallXsY0n5XhZt2cuTC7fy8LzNAAzpnc0pg3sxZVg+pwzOU41BJEY1AunSwhHHym37mLOuirnrd/Huxt3UNoTJSAny6aF5nDWigHNGFeo5g3R5R6sRKBFIUjnUGGbeht28umIHr67cwbbqOlJDAc4ZWcAVE/vyqSF56qEkXZISgUgTnHMsLq/m8QXlPL14K3sPNlCQk8YXTxnI507qT066agnSdSgRiBzDocYws1bt5OF5m3lrXRXd0kJcc1J/bpg8kN56yCxdgBKBSCssq6jm3jfW8/zSbaSFgtxy5nHccOpAUkNqMpLO62iJQD/ZIkcYXZzLPddMYNY3p/DpoXn8/MVVXHD3m8zboHcdpWtSIhBpxoBeWfzftaU88IVSahvCTLtvHt95fGm7B/wTSTR6j0DkGM4cUcApg/O469U13Dd7A+t37ucPn59Ar+w0v0MT6RCqEYi0QEZqkO+eP4K7rx7P4vK9XPK7OazeXuN3WCIdQolApBUuHlvE3798MvWNEa74w1zmrGt2oFyRTkOJQKSVxvXrzlNfnUxx9wy+/Jf5rNr+iWk2RDoVJQKRNuiTm8GML51AVlqQL/35PXbsq/M7JJE2UyIQaaM+uRk88IUT2FvbwA0PvsfB+ka/QxJpEyUCkXaIvnMwnhVb93HLowsJRzrXC5oioEQg0m5nDC/gvy4exasrd3LvG+v9Dkek1TxLBGbWz8xmmdkKM1tuZrc2UeZzZrbEzJaa2VwzG+tVPCJeuu7kEqaOKuTuf61lU9UBv8MRaRUvawSNwDeccyOBk4CbzWzkEWU2Aqc5544Hbgfu8zAeEU/9+JJRpAYDfO/JpXS2MbwkuXmWCJxz25xzC2LrNcBKoPiIMnOdc3tiH+cBfb2KR8RrBTnpfPu84cxZt4uZCyr8DkekxeLyjMDMSoDxwDtHKXYD8EI84hHxyjWT+jNxQA/ueG4Fu/Yf8jsckRbxPBGYWTYwE7jNOdfkmzdmdjrRRPDtZvbfaGZlZlZWWVnpXbAi7RQIGD+9/HgOHGrkjudW+h2OSIt4mgjMLIVoEvirc+7xZsqMAe4HLnHONTnOr3PuPudcqXOuND8/37uARTrA0IJu3HTaYJ5YWEHZpt1+hyNyTF72GjLgAWClc+7OZsr0Bx4HrnXOrfEqFpF4u2nKEPKy07jrVf1YS+LzskYwGbgWOMPMFsWW881suplNj5X5IdAL+H1sv6Yeky4hIzXITVMGM2fdLk1oIwlPU1WKeKSuIcynfzGLkrws/n7jSUQrySL+0FSVIj5ITwly8+lDeHfjbt5er1qBJC4lAhEPXXVCP/rkpnPnK2v0kpkkLCUCEQ99UCsoe38Pb67VJDaSmJQIRDz22dJ+FHfPUK1AEpYSgYjHUkMBvnrGEBZt2cts1QokASkRiMTBFRP6Utw9g9+8qlqBJB4lApE4SA0FmD5lMAs272WuehBJglEiEImTz5b2pTAnnbv/tdbvUEQ+RolAJE7SQkG+fNog3tm4W28bS0JRIhCJo6sn9ScvO43fvqZagSQOJQKROEpPCTL9tEHMWbdLI5NKwlAiEImza07sT6+sVO6Ztc7vUEQAJQKRuMtMDfGFU0p4fXUl63bu9zscESUCET9cc2J/UoMBHpy7ye9QRJQIRPyQl53GRWOLmLmgnOraBr/DkSSnRCDik+snl3CwPsw/yrb4HYokOS+nquxnZrPMbIWZLTezW5soM9zM3jazQ2b2Ta9iEUlEo4tzmVTSkxlzNxGOaNgJ8Y+XNYJG4BvOuZHAScDNZjbyiDK7gVuAX3kYh0jCun5yCeV7anl15Q6/Q5Ek5lkicM5tc84tiK3XACuB4iPK7HTOvQeokVSS0tkjCyjunsGf52z0OxRJYnF5RmBmJcB44J02Hn+jmZWZWVllZWVHhibiq1AwwLUnD2Deht2s3LbP73AkSXmeCMwsG5gJ3Oaca9NPunPuPudcqXOuND8/v2MDFPHZtBP6kZ4S4C/z3vc7FElSniYCM0shmgT+6px73MtriXRW3TNTOXdUIc8t2UZ9Y8TvcCQJedlryIAHgJXOuTu9uo5IV3Dp+GKqaxt4ffVOv0ORJBTy8NyTgWuBpWa2KLbtu0B/AOfcvWZWCJQBOUDEzG4DRra1CUmks/rUkDx6ZaXy5KIKzhlV6Hc4kmQ8SwTOubcAO0aZ7UBfr2IQ6SxCwQAXjS3ikXc3U13bQG5Git8hSRLRm8UiCeKy8cXUN0Z4cdk2v0ORJKNEIJIgxvTNZVBeFk8srPA7FEkySgQiCcLMuHR8MfM27KZib63f4UgSUSIQSSCXjCsC4OlFW32ORJKJEoFIAhnQK4sJ/bvzxMJynNNAdBIfSgQiCeay8cWs2bGfldtq/A5FkoQSgUiCuWBMEcGA8cwSNQ9JfCgRiCSYnlmpnDK4F88t2abmIYkLJQKRBHThmD5s3n2Q5Vv1kr14T4lAJAGdM7KQUMB4doleLhPvKRGIJKAeWalMHpLHc0u3qnlIPKdEIJKgLhjThy27a1laUe13KJIA1uyo8WxuayUCkQR1zsgCQgHjOTUPJb3KmkN85g9z+fEzyz05vxKBSILqnpnKqcfl8dxS9R5Kdnc8t4K6hgjXnVziyfmVCEQS2AXH96F8Ty1LytU8lKxmr6nkqUVbuWnKYIb0zvbkGkoEIgnsnJGFpASN55aqeSgZ1TWE+cFTyxiUl8VNUwZ7dh0vp6rsZ2azzGyFmS03s1ubKGNmdreZrTOzJWY2wat4RDqj3MwUPnVcvl4uS1L3vLaO93cd5I7LRpOeEvTsOl7WCBqBbzjnRgInATeb2cgjypwHHBdbbgT+4GE8Ip3SBcf3oWJvLQu37PU7FImjtTtq+L/Z67l8QjGnDM7z9FqeJQLn3Dbn3ILYeg2wEig+otglwEMuah7Q3cz6eBWTSGd09qgCUoMB9R5KIjV1Dfz7Y4vISgvxvfNHeH69uDwjMLMSYDzwzhG7ioEth30u55PJAjO70czKzKyssrLSqzBFElJOegqfHhptHop41I9cEsfB+ka+NOM9Vm2r4ddXjqVXdprn1/Q8EZhZNjATuM0516aBU5xz9znnSp1zpfn5+R0boEgncNHYPmzfV0fZ+3v8DkU8VNcQ5t8eKmP++3v432njOHNEQVyu62kiMLMUokngr865x5soUgH0O+xz39g2ETnMmSMKSAsFeFZDU3dZ9Y0RvvLXBcxZt4tffmYsF44pitu1vew1ZMADwErn3J3NFHsauC7We+gkoNo5p4ZQkSNkp4U4Y3hvnl+63bNhBsQ/kYjjG/9YzGurdvKTy0ZzxcS+cb2+lzWCycC1wBlmtii2nG9m081seqzM88AGYB3wR+ArHsYj0qldOKaIqv2HeGfDLr9DkQ72P8+v5JnFW/n21OF87sQBcb9+yKsTO+feAuwYZRxws1cxiHQlZwzvTWZqkGeWbOOUId52J5T4uf/NDdz/1ka+eEoJ008b5EsMerNYpJPISA1y5ogCXly2jYZwxO9wpAM8vXgrdzy3kvNGF/KDC0cSbVGPPyUCkU7kwjF92HOwgbnr1TzU2c3bsItvPraYSSU9ueuqcQQD/iQBUCIQ6VROG5pPt7QQzy5W76HObFPVAaY/PJ9+PTP443Wlng4f0RJKBCKdSHpKkLNHFvDisu3sPlDvdzjSBtUHG/jSg+9hwJ++eAK5mSl+h6REINLZfPm0wdQ2hLn92RV+hyKt1BCO8JVH5rNl90H+79pSBvTK8jskQIlApNMZVtiNr5w+hCcWVvD66p1+hyMt5JzjR08vZ866XfzPZcczaWBPv0P6kBKBSCd08+nRSUq+98Qy9h9q9DscaYEH527ikXc2M/20wVxZ2u/YB8SREoFIJ5QWCvLzK8awtbqWX7202u9w5BjeWlvF7c+t5OyRBfzHucP8DucTlAhEOqmJA3rwhZNLePDtTczXYHQJa2PVAW5+ZAGD87O466pxBHzsJtqcFiUCMxtsZmmx9SlmdouZdfc2NBE5lm+dO4yi3Ay+9Y/FaiJKQPvqGvi3h8oIGNx/3Qlkp3k2mEO7tLRGMBMIm9kQ4D6iI4Y+4llUItIiWWkhfv3ZsWzadYDvP7FU01kmkEjEcdvfFrGp6gC//9xE+vfK9DukZrU0EUScc43AZcBvnXPfAjSTmEgCOGlQL247ayhPLtrKY2Vbjn2AxMXdr63ltVU7+dFFIzl5cC+/wzmqliaCBjO7GvgC8Gxsm/9vQYgIADefPoRTh+Txw6eWs2p7m+Z/kg40a9VOfvOvtVwxoS+fPyn+o4m2VksTwfXAycBPnHMbzWwg8BfvwhKR1ggGjLuuGkdORgo3/3UBB/S8wDebdx3k1r8tZERhDj+5bLRvA8m1RosSgXNuhXPuFufco2bWA+jmnPu5x7GJSCvkd0vjN9PGsbHqAHe9ssbvcJJSXUOY6Q/PB+Dez0/0fQyhlmppr6HXzSzHzHoCC4A/mllzs46JiE9OGZzH+cf34R/zy6lrCPsdTtL54VPLWLFtH7+ZNj6hHw4fqaVNQ7mxiecvBx5yzp0InHW0A8zsT2a208yWNbO/h5k9YWZLzOxdMxvdutBFpClXndCP6toGXl25w+9QksrM+eU8VlbOLWcM4fThvf0Op1VamghCZtYH+CwfPSw+lhnA1KPs/y6wyDk3BrgO+E0LzysiR3HK4DyKu2fw9/fUgyhe1u3czw+eWsaJA3ty61lD/Q6n1VqaCP4beAlY75x7z8wGAWuPdoBzbjaw+yhFRgKvxcquAkrMrKCF8YhIM4IB44qJfXlrXRUVe2v9DqfLq2sI89VHFpCeEuQ308b7OsFMW7X0YfE/nHNjnHM3xT5vcM5d0c5rLyba1ISZTQIGAH2bKmhmN5pZmZmVVVZWtvOyIl3flRP74ly0uUK8dfuzK1i1vYY7PzuWwtx0v8Npk5Y+LO4ba8/fGVtmmlmTv7Rb4WdAdzNbBHwNWAg0+XTLOXefc67UOVean5/fzsuKdH39emYyeUgv/jF/C5GI3jb2yrNLtvLX2IiiU4Z1rucCh2tp09CfgaeBotjyTGxbmznn9jnnrnfOjSP6jCAf2NCec4rIRz5b2o8tu2uZt0HzG3uhYm8t33l8KeP6decb53S+5wKHa2kiyHfO/dk51xhbZhD9xd1mZtbdzFJjH/8fMDvWM0lEOsC5owrJSQ9p2AkPhCOOr/99EZGI4zfTxpES7NwDObc0+l1m9nkzC8aWzwNH/TPDzB4F3gaGmVm5md1gZtPNbHqsyAhgmZmtBs4Dbm3rP0JEPik9Jcgl44p5Ydl2qmsb/A6nS7lv9gbe2bibH108KmGmm2yPlo6J+iXgt8BdgAPmAl882gHOuauPsf9toHPXp0QS3GdL+/GXee/z3JJtXHNif7/D6RKWVVRz5yurOW90IVdObO+j0sTQ0l5D7zvnLnbO5TvnejvnLgXa22tIRDw2ujiHkl6ZvLh8u9+hdAm19WFu+dtCemal8j+XHd8pxhFqifY0bH29w6IQEU+YGeeOLmTuuio1D3WAn7+4ig2VB/j1lePokZV67AM6ifYkgq6RCkW6uKmjCmmMOF5bpSEn2uPt9buYMXcTXzylhFOPy/M7nA7VnkSgzskincDYvt0pyEnjpWVKBG114FAj3/rnYgb0yuQ/pibe5PPtddSHxWZWQ9O/8A3I8CQiEelQgYBx7qhCHivbQm19mIzUzjE0ciL56Qsrqdhby2NfPpnM1MScd7g9jlojcM51c87lNLF0c851vbsh0kVNHVVIXUOEN9ZoiJbWemttFQ/P28wNkwdyQklPv8PxROd+C0JEWmTSwJ50z0zhJfUeapX9hxr59swlDMrL4pvndr0moQ8oEYgkgVAwwNkjCnh15Q7qGyN+h9Np/PLFVWytruWXV47tNLONtYUSgUiSmDq6kJq6Rt7W2EMtMv/9PTw0732+cHIJEwf08DscTykRiCSJyUPyyEoN8uIyNQ8dS31jhP+cuYQ+OelduknoA0oEIkkiPSXI6cN788qK7YQ1NPVR/eH19azduZ87LhtNdlrX7xejRCCSRKaOLqRqfz1vravyO5SEtW5nDb+btY4Lx/ThjOHJMWmiEoFIEjl7ZAF52WnMmLPR71ASUiTi+M7jS8lIDfKji0b5HU7cKBGIJJG0UJBrTxrArNWVrK/c73c4Ceef88t5b9Mevnf+CPK7pfkdTtwoEYgkmc+d1J/UYIAZczb5HUpC2Xuwnp+9uIqJA3rwmS4yvHRLKRGIJJm87DQuGVfEP+eXU31QI5J+4BcvrWbvwXpuv2Q0gUByjanpWSIwsz/FJrpf1sz+XDN7xswWm9lyM7veq1hE5OOunzyQ2oYwf3tvs9+hJITFW/by6Lub+cIpJYwsyvE7nLjzskYwA5h6lP03Ayucc2OBKcCvD5vDWEQ8NLIoh5MH9eLBuZtoDCf3m8bhiOP7Ty4jLzuNfz87OSdN9CwROOdmA7uPVgToZtEpfrJjZRu9ikdEPu5Lpw5ka3UdLy1P7uGpH3l3M0srqvn+BSPISU/xOxxf+PmM4B6iE9hvBZYCtzrnmvzTxMxuNLMyMyurrNToiSId4YzhvRnQK5MH3trgdyi+2XOgnl+9tJqTBvXk4rFFfofjGz8TwbnAIqAIGAfcY2ZNNs455+5zzpU650rz8/PjGaNIlxUMGF+aPJAFm/cyJ0lfMPvfV9dQU9fAf108qsvMP9wWfiaC64HHXdQ6YCMw3Md4RJLOtEn9KMpN5xcvrca55Bp2Ys2OGh5+ZzPXnNif4YXJ94D4cH4mgs3AmQBmVgAMA5K3jirig7RQkNvOGsriLXt5eUXyPCtwznH7syvISg3y9bO7/qByx+Jl99FHgbeBYWZWbmY3mNl0M5seK3I7cIqZLQX+BXzbOZec9VMRH10+oZhB+Vn86qXVSTMY3WurdvLm2ipuO2soPbPUWdGzYfWcc1cfY/9W4Byvri8iLRMKBvjG2cO4+ZEFPLWogssndO23ausbI9zx3EoG52dx7ckD/A4nIejNYhHhvNGFjCrK4a5X13T5GcweensTG6sO8P0LR5IS1K9AUCIQESAQML517jC27K7l7134beO9B+u5+19rOW1oPqcP6+13OAlDiUBEADhtaD6TSnryu1nraeiibxv//vX11Bxq5D/PUwfFwykRiAgAZsb0KYPYvq+O55du8zucDle+5yAz5m7iigl9GdEnubuLHkmJQEQ+NGVobwblZ3H/mxu73HsFd768BgO+nqTjCR2NEoGIfCgQe9t4aUU1723a43c4HWb51mqeWFTB9ZMHUtQ9w+9wEo4SgYh8zBUT+tI9M4X73+w673f+7IVV5GakcNOUwX6HkpCUCETkYzJSg3zuxP68snIH7+864Hc47fbW2ireXFvFV08fQm5Gco4ueixKBCLyCdedXEIoYPy5k09n6Zzjly+vprh7hl4eOwolAhH5hIKcdC4aW8RjZVuoru2801n+a+VOFm/Zy61nHkdaKOh3OAlLiUBEmnTDqQM5WB/mb+92zhfMIhHHr19ZQ0mvTC6fUOx3OAlNiUBEmjSqKJcTSnrwWNmWTtmV9IVl21m5bR+3nTWUkIaSOCrdHRFp1uUT+rK+8gBLK6r9DqVVwhHHna+s5rje2VyUxDOPtZQSgYg06/zj+5AaCvD4ggq/Q2mVpxZVsL7yAF8/eyjBQPLOPNZSSgQi0qzcjBTOHlHA04u3dprxhxrCEf731bWMKsrh3FGFfofTKSgRiMhRXT6hmN0H6nljdaXfobTIEwsq2Lz7IF8/eygB1QZaxMsZyv5kZjvNbFkz+79lZotiyzIzC5tZT6/iEZG2+fTQfHplpfLEwsRvHmoMR/jd6+s4vjiXM4ZrmOmW8rJGMAOY2txO59wvnXPjnHPjgO8AbzjndnsYj4i0QUowwEVji3hl5Y6Ef6fgmSVbeX/XQb56xhDMVBtoKc8SgXNuNtDSX+xXA496FYuItM/lE4qpb4wk9PDU4Yjjt6+tY3hhN84eUeB3OJ2K788IzCyTaM1h5lHK3GhmZWZWVlnZOdopRbqS44tzGdI7m8cXlPsdSrOeX7qNDZUH+NoZx+nZQCv5ngiAi4A5R2sWcs7d55wrdc6V5ufnxzE0EYHopDWXjS/mvU172LzroN/hfEIk4rjntXUM6Z3NeaPVU6i1EiERTEPNQiIJ77LxxZiRkA+NX16xg9U7avjq6UNUG2gDXxOBmeUCpwFP+RmHiBxbUfcMThzYkycXVSTUkBPOOe6ZtZaSXplcOKaP3+F0Sl52H30UeBsYZmblZnaDmU03s+mHFbsMeNk51/kHPRdJApeP78vGqgMsLk+cISdmr61iWcU+vjJliMYUaqOQVyd2zl3dgjIziHYzFZFOYOrxhXz/qWU8saCccf26+x0OAPe+vp7CnHQuHa8RRttK6VNEWiwnPTrkxDNLtiXEkBOLt+zl7Q27uOHUgaSG9OusrXTnRKRVLh0fHXLizbX+d+W+9431dEsPMW1SP79D6dSUCESkVU4bmk+PzBSeWLjV1zg2VO7nxeXbufakAXRL11zE7aFEICKtkhoKcOGYIl5evp2aOv+GnPjjmxtJCQa4fvJA32LoKpQIRKTVLh1fzKHGCC8u2+7L9XfW1DFzQTmfmdiX/G5pvsTQlSgRiEirTejfnQG9MnlykT8vl/15ziYawhFu/NQgX67f1SgRiEirmRmXjitm7vpdbK+ui+u1DzWGefTdzZw7spCSvKy4XrurUiIQkTa5eFwRzsELy+I7IumsVTvZe7CBq9RTqMMoEYhImwzOz2ZYQTdeWBrf5wQzF1SQ3y2NTw3Ji+t1uzIlAhFps6mjC3nv/d3srIlP89Cu/YeYtWonl40v1nASHUh3UkTa7Pzj++AcvLR8R1yu9/TirTRGHJdP0HASHUmJQETabGhBNoPysngxTs8JHl9QwaiiHIYX5sTleslCiUBE2szMOO/4QuZt2M3uA/WeXmvNjhqWVlRz+YS+nl4nGSkRiEi7nDe6D+GI45UV3j40nrmgnGDAuGRckafXSUZKBCLSLqOKcujXM4PnPew9FI44nlxYwZSh+eRl603ijublxDR/MrOdZrbsKGWmmNkiM1tuZm94FYuIeMfMOG90H+aur6L6oDdjD81ZV8WOfYe4YqKahbzgZY1gBjC1uZ1m1h34PXCxc24UcKWHsYiIh84bXUhD2PHqSm96Dz2xsIKc9BBnjujtyfmTnWeJwPbZy/wAAApNSURBVDk3G9h9lCLXAI875zbHyu/0KhYR8dbYvt3pk5vOCx4MQneoMcwrK3YwdXQhaaFgh59f/H1GMBToYWavm9l8M7vOx1hEpB0CAWPq6EJmr63s8KGp31xTxf5DjZx3vCam94qfiSAETAQuAM4FfmBmQ5sqaGY3mlmZmZVVVvo/K5KIfNKFY4qob4zwcge/XPb8sm3kpIeYPFhDSnjFz0RQDrzknDvgnKsCZgNjmyronLvPOVfqnCvNz8+Pa5Ai0jIT+nenuHsGTy/uuJnL6hsjvLJiB2ePLNScxB7y884+BZxqZiEzywROBFb6GI+ItIOZcdHYIt5aV8Wu/Yc65Jxz1lVRU9fIBWMKO+R80jQvu48+CrwNDDOzcjO7wcymm9l0AOfcSuBFYAnwLnC/c67ZrqYikvguHltEOOJ4voMeGj+/dBvd0kJM1kijngp5dWLn3NUtKPNL4JdexSAi8TWiTzeG9M7mmUVbufakAe06V0M4wssrdnDWyAL1FvKYGt1EpMOYGRePLeLdTbvZure2Xeeau34X1bUNnK/eQp5TIhCRDnXx2OhYQM8uad9D4xeWbiMrNcinjlOzkNeUCESkQ5XkZTGmb267eg81hCO8tHw7Z44oID1FzUJeUyIQkQ538dgillXsY0Pl/jYd/86G3ew5qGaheFEiEJEOd+GYIsxoc63g2SVbyUoNctpQvTcUD0oEItLhCnPTmVTSkycXVuCca9WxhxrDPL90G+eOKiQjVc1C8aBEICKeuOqEfmzadZC563e16rg3Vleyr66RizUBTdwoEYiIJ84/vg/dM1N4eN77rTruqcVb6ZWVqpfI4kiJQEQ8kZ4S5MqJfXl5xQ527Ktr0TH7DzXy6oodXDCmDylB/XqKF91pEfHMNScOIBxx/P29LS0q//Ly7RxqjGhe4jhTIhARzwzMy+JTx+Xx6LubaQxHjln+yUVb6dsjgwn9e8QhOvmAEoGIeOpzJ/ZnW3Uds1YffS6RyppDzFlXxcVjizCzOEUnoEQgIh47a0QBBTlpx3xo/PzSbYQjjkvGFccpMvmAEoGIeCoUDDDthP7MXlvJ5l0Hmy331KIKhhd2Y1hhtzhGJ6BEICJxMG1SPwJm/PiZ5VTWfHzSGuccLy7bxoLNe/XugE+UCETEc31yM/jGOUN5Y00lZ/zqde5/cwP1jRGWlldz9R/nMf3hBQzOz+LKif38DjUpWWtf/27xic3+BFwI7HTOjW5i/xSi01VujG163Dn338c6b2lpqSsrK+vIUEUkTtZX7uf2Z1fw+upKCnLS2LHvED2zUvn3s45j2qT+enfAQ2Y23zlX2tQ+z2YoA2YA9wAPHaXMm865Cz2MQUQSyOD8bGZcP4nXVu3g/jc3cvmEvtw0ZTA56Sl+h5bUvJyqcraZlXh1fhHpvM4YXsAZwwv8DkNi/K6HnWxmi83sBTMb5XMsIiJJycumoWNZAAxwzu03s/OBJ4HjmipoZjcCNwL0798/fhGKiCQB32oEzrl9zrn9sfXngRQza3K4Qefcfc65UudcaX6+JqoQEelIviUCMyu02HvkZjYpFkvrBi4XEZF286xpyMweBaYAeWZWDvwISAFwzt0LfAa4ycwagVpgmvOqL6uIiDTLy15DVx9j/z1Eu5eKiIiP/O41JCIiPlMiEBFJcp4NMeEVM6sEWjcJ6kdygeoOLn+sMs3tb832I7flAVXHiKujtfbetfd43fvmY/D6eN375mPw+viWlj9aueb2DXDONd3t0jmXNAtwX0eXP1aZ5va3ZvuR24CyRL93uve697r33tz7Y5VrS8zJ1jT0jAflj1Wmuf2t2d7auL3Q3hh079tO994/iXjvj1Wu1TF3uqYhATMrc82MIije0r33j+69d5KtRtBV3Od3AElM994/uvceUY1ARCTJqUYgIpLklAhERJKcEoGISJJTIuhizOxSM/ujmf3dzM7xO55kYmaDzOwBM/un37EkAzPLMrMHYz/vn/M7ns5MiSCBmNmfzGynmS07YvtUM1ttZuvM7D+Pdg7n3JPOuX8DpgNXeRlvV9JB936Dc+4GbyPt2lr5fbgc+Gfs5/3iuAfbhSgRJJYZwNTDN5hZEPgdcB4wErjazEaa2fFm9uwRS+/DDv1+7DhpmRl03L2XtptBC78PQF9gS6xYOI4xdjl+TlUpR3DOzTazkiM2TwLWOec2AJjZ34BLnHM/BS488hyxyX5+BrzgnFvgbcRdR0fce2m/1nwfgHKiyWAR+qO2XXTzEl8xH/3VA9Ef/uKjlP8acBbwGTOb7mVgSaBV997MepnZvcB4M/uO18Elkea+D48DV5jZH0iM4Sg6LdUIuhjn3N3A3X7HkYycc7uIPpuROHDOHQCu9zuOrkA1gsRXAfQ77HPf2Dbxnu59YtD3wWNKBInvPeA4MxtoZqnANOBpn2NKFrr3iUHfB48pESQQM3sUeBsYZmblZnaDc64R+CrwErASeMw5t9zPOLsi3fvEoO+DPzTonIhIklONQEQkySkRiIgkOSUCEZEkp0QgIpLklAhERJKcEoGISJJTIpAuw8z2x/l698dGwYznNW8zs8x4XlO6Pr1HIF2Gme13zmV34PlCsZeZ4iY2eqw55yLN7N8ElDrnquIZl3RtqhFIl2Zm+WY208zeiy2TY9snmdnbZrbQzOaa2bDY9i+a2dNm9hrwLzObYmavm9k/zWyVmf019sua2PbS2Pp+M/uJmS02s3lmVhDbPjj2eamZ3dFUrcXMSmKTrjwELAP6mdkfzKzMzJab2Y9j5W4BioBZZjYrtu2c2L9jgZn9w8w6LBFKEnHOadHSJRZgfxPbHgFOja33B1bG1nOAUGz9LGBmbP2LRIc57hn7PAWoJjrQWYDo8AcfnO91on+dAzjgotj6L4Dvx9afBa6OrU9vJsYSIAKcdNi2D64fjF1nTOzzJiAvtp4HzAayYp+/DfzQ7++Dls63aBhq6erOAkbG/ogHyIn91ZwLPGhmxxH9JZ5y2DGvOOd2H/b5XedcOYCZLSL6i/utI65TT/SXPsB84OzY+snApbH1R4BfNRPn+865eYd9/qyZ3Uh0qPg+RGfmWnLEMSfFts+J/ftSiSYqkVZRIpCuLkD0L+26wzea2T3ALOfcZbEZsV4/bPeBI85x6LD1ME3/v2lwzrljlDmaD69pZgOBbwInOOf2mNkMIL2JY4xo0rq6ldcS+Rg9I5Cu7mWis7YBYGbjYqu5fDSm/Rc9vP484IrY+rQWHpNDNDFUx541nHfYvhqg22HnnmxmQwDMLMvMhrY/ZEk2SgTSlWTGhi7+YPk6cAtQamZLzGwFH80g9gvgp2a2EG9rxrcBXzezJcAQos8bjso5txhYCKwi2pw057Dd9wEvmtks51wl0ST2aOz8bwPDOzZ8SQbqPirioVif/1rnnDOzaUQfHF/id1wih9MzAhFvTQTuiXU53Qt8yed4RD5BNQIRkSSnZwQiIklOiUBEJMkpEYiIJDklAhGRJKdEICKS5JQIRESS3P8HBNeO4iCpsHAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "reset_seed(SEED)\n",
    "batch_size = 8\n",
    "accumulation_steps = DESIRED_BATCH_SIZE//batch_size\n",
    "\n",
    "# re-create `DataLoader` to apply the new `batch_size`\n",
    "trainloader = DataLoader(trainset, batch_size=batch_size, shuffle=True)\n",
    "testloader = DataLoader(testset, batch_size=DESIRED_BATCH_SIZE*2, shuffle=False)\n",
    "\n",
    "device = torch.device('cuda')\n",
    "model = ConvNet()\n",
    "model = model.to(device)\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.5)\n",
    "criterion = nn.NLLLoss()\n",
    "\n",
    "lr_finder = LRFinder(model, optimizer, criterion, device='cuda')\n",
    "lr_finder.range_test(trainloader, val_loader=testloader, end_lr=10, num_iter=100, step_mode='exp', accumulation_steps=accumulation_steps)\n",
    "lr_finder.plot()\n",
    "lr_finder.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch-lr-finder3",
   "language": "python",
   "name": "pytorch-lr-finder3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
